[clblas] 22/67: adding auto-gemm script
Ghislain Vaillant
ghisvail-guest at moszumanska.debian.org
Tue Oct 27 08:02:11 UTC 2015
This is an automated email from the git hooks/post-receive script.
ghisvail-guest pushed a commit to branch master
in repository clblas.
commit 40098f405f690b71c896bc482818ea435c50eacb
Author: David Tanner <guacamoleo at gmail.com>
Date: Thu Jul 9 13:35:45 2015 -0500
adding auto-gemm script
---
.gitignore | 3 +
src/CMakeLists.txt | 85 +-
src/client/clfunc_common.hpp | 18 +-
src/client/clfunc_xgemm.hpp | 192 ++-
src/client/clfunc_xgemv.hpp | 22 +-
src/client/clfunc_xger.hpp | 16 +-
src/client/clfunc_xgerc.hpp | 12 +-
src/client/clfunc_xgeru.hpp | 12 +-
src/client/clfunc_xhemm.hpp | 34 +-
src/client/clfunc_xhemv.hpp | 12 +-
src/client/clfunc_xher.hpp | 10 +-
src/client/clfunc_xher2.hpp | 12 +-
src/client/clfunc_xher2k.hpp | 20 +-
src/client/clfunc_xherk.hpp | 20 +-
src/client/clfunc_xsymm.hpp | 58 +-
src/client/clfunc_xsymv.hpp | 12 +-
src/client/clfunc_xsyr.hpp | 10 +-
src/client/clfunc_xsyr2.hpp | 12 +-
src/client/clfunc_xsyr2k.hpp | 34 +-
src/client/clfunc_xsyrk.hpp | 32 +-
src/client/clfunc_xtrmm.hpp | 48 +-
src/client/clfunc_xtrmv.hpp | 14 +-
src/client/clfunc_xtrsm.hpp | 50 +-
src/client/clfunc_xtrsv.hpp | 14 +-
src/client/client.cpp | 10 +-
src/library/CMakeLists.txt | 400 +++++-
src/library/blas/AutoGemm/.gitignore | 4 +
src/library/blas/AutoGemm/AutoGemm.py | 47 +
src/library/blas/AutoGemm/AutoGemmParameters.py | 149 +++
.../AutoGemmTools/AutoGemmPreCompileKernels.cpp | 924 +++++++++++++
.../blas/AutoGemm/AutoGemmTools/AutoGemmUtil.h | 793 +++++++++++
.../AutoGemm/AutoGemmTools/ProfileAutoGemm.cpp | 1378 ++++++++++++++++++++
.../blas/AutoGemm/AutoGemmTools/TestAutoGemm.cpp | 995 ++++++++++++++
src/library/blas/AutoGemm/Common.py | 60 +
src/library/blas/AutoGemm/Includes.py | 458 +++++++
src/library/blas/AutoGemm/KernelOpenCL.py | 549 ++++++++
src/library/blas/AutoGemm/KernelParameters.py | 253 ++++
src/library/blas/AutoGemm/KernelSelection.py | 683 ++++++++++
src/library/blas/AutoGemm/KernelsToPreCompile.py | 91 ++
src/library/blas/AutoGemm/README.txt | 0
.../UserGemmKernelSources/UserGemmClKernels.h | 18 +
.../UserGemmKernelSourceIncludes.cpp | 57 +
.../UserGemmKernelSourceIncludes.h | 80 ++
.../dgemm_Col_NN_B0_MX048_NX048_KX08_src.cpp | 203 +++
.../dgemm_Col_NN_B1_MX048_NX048_KX08_src.cpp | 203 +++
.../dgemm_Col_NT_B0_MX048_NX048_KX08_src.cpp | 196 +++
.../dgemm_Col_NT_B1_MX048_NX048_KX08_src.cpp | 193 +++
.../dgemm_Col_TN_B0_MX048_NX048_KX08_src.cpp | 195 +++
.../dgemm_Col_TN_B1_MX048_NX048_KX08_src.cpp | 195 +++
.../sgemm_Col_NN_B0_MX032_NX032_KX16_src.cpp | 129 ++
.../sgemm_Col_NN_B0_MX064_NX064_KX16_src.cpp | 160 +++
.../sgemm_Col_NN_B0_MX096_NX096_KX16_src.cpp | 208 +++
...sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src.cpp | 149 +++
.../sgemm_Col_NN_B1_MX032_NX032_KX16_src.cpp | 129 ++
.../sgemm_Col_NN_B1_MX064_NX064_KX16_src.cpp | 161 +++
.../sgemm_Col_NN_B1_MX096_NX096_KX16_src.cpp | 207 +++
.../sgemm_Col_NT_B0_MX032_NX032_KX16_src.cpp | 126 ++
.../sgemm_Col_NT_B0_MX064_NX064_KX16_src.cpp | 165 +++
.../sgemm_Col_NT_B0_MX096_NX096_KX16_src.cpp | 210 +++
...sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src.cpp | 148 +++
...sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src.cpp | 158 +++
.../sgemm_Col_NT_B1_MX032_NX032_KX16_src.cpp | 126 ++
.../sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src.cpp | 161 +++
.../sgemm_Col_NT_B1_MX064_NX032_KX16_COL_src.cpp | 157 +++
.../sgemm_Col_NT_B1_MX064_NX064_KX16_src.cpp | 160 +++
.../sgemm_Col_NT_B1_MX096_NX096_KX16_src.cpp | 208 +++
.../sgemm_Col_NT_B1_MX128_NX128_KX16_src.cpp | 290 ++++
.../sgemm_Col_TN_B0_MX032_NX032_KX16_src.cpp | 128 ++
.../sgemm_Col_TN_B0_MX064_NX064_KX16_src.cpp | 165 +++
.../sgemm_Col_TN_B0_MX096_NX096_KX16_src.cpp | 209 +++
...sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src.cpp | 148 +++
.../sgemm_Col_TN_B1_MX032_NX032_KX16_src.cpp | 127 ++
.../sgemm_Col_TN_B1_MX064_NX064_KX16_src.cpp | 165 +++
.../sgemm_Col_TN_B1_MX096_NX096_KX16_src.cpp | 209 +++
src/library/blas/include/xgemm.h | 39 +
src/library/blas/specialCases/GemmSpecialCases.cpp | 829 ++++++++++++
.../blas/specialCases/include/GemmSpecialCases.h | 42 +
src/library/blas/xgemm.cc | 855 ++++++++----
src/tests/CMakeLists.txt | 2 +-
src/tests/common.cpp | 29 +-
src/tests/correctness/corr-gemm.cpp | 12 +-
src/tests/include/gemm.h | 6 +-
82 files changed, 14273 insertions(+), 630 deletions(-)
diff --git a/.gitignore b/.gitignore
index 59cf9c1..b0ad3e7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,6 @@
# flags.txt file
*flags.txt
+
+# vim temp files
+.*.swp
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 9219c98..fe61ba5 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -26,52 +26,55 @@ option( BUILD_KTEST "A command line tool for testing single clBLAS kernel" ON )
option( BUILD_SHARED_LIBS "Build shared libraries" ON )
#enable or disable offline compilation for different devices. Currently only Hawaii, Bonaire, Tahiti have the option.
-option( OCL_OFFLINE_BUILD_HAWAII_KERNEL "Offline compile the OpenCL kernels for Hawaii device" OFF)
-option( OCL_OFFLINE_BUILD_BONAIRE_KERNEL "Offline compile the OpenCL kernels for Bonaire device" OFF)
-option( OCL_OFFLINE_BUILD_TAHITI_KERNEL "Offline compile the OpenCL kernels for Tathit device" OFF)
-
-if( (OCL_OFFLINE_BUILD_HAWAII_KERNEL AND OCL_OFFLINE_BUILD_BONAIRE_KERNEL) OR (OCL_OFFLINE_BUILD_HAWAII_KERNEL AND OCL_OFFLINE_BUILD_TAHITI_KERNEL) OR (OCL_OFFLINE_BUILD_BONAIRE_KERNEL AND OCL_OFFLINE_BUILD_TAHITI_KERNEL))
- MESSAGE( WARNING "More than one device is chosen for offline compilation of static kernels. This might result in running out of heap memory with certain driver. Please consider offline compliation for ONE device only." )
-endif( )
-
-if( NOT OCL_OFFLINE_BUILD_HAWAII_KERNEL )
+#option( OPENCL_OFFLINE_BUILD_HAWAII_KERNEL "Offline compile the OpenCL kernels for Hawaii device" OFF)
+#option( OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL "Offline compile the OpenCL kernels for Bonaire device" OFF)
+#option( OPENCL_OFFLINE_BUILD_TAHITI_KERNEL "Offline compile the OpenCL kernels for Tathit device" OFF)
+set( OPENCL_OFFLINE_BUILD_HAWAII_KERNEL OFF)
+set( OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL OFF)
+set( OPENCL_OFFLINE_BUILD_TAHITI_KERNEL OFF)
+
+#if( (OPENCL_OFFLINE_BUILD_HAWAII_KERNEL AND OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL) OR (OPENCL_OFFLINE_BUILD_HAWAII_KERNEL AND OPENCL_OFFLINE_BUILD_TAHITI_KERNEL) OR (OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL AND OPENCL_OFFLINE_BUILD_TAHITI_KERNEL))
+# MESSAGE( WARNING "More than one device is chosen for offline compilation of static kernels. This might result in running out of heap memory with certain driver. Please consider offline compliation for ONE device only." )
+#endif( )
+
+#if( NOT OPENCL_OFFLINE_BUILD_HAWAII_KERNEL )
#use dynamic generated kernels
- MESSAGE(STATUS "Build dynamic Hawaii kernels.")
- MESSAGE(STATUS "Check OCL_OFFLINE_BUILD_HAWAII_KERNEL to build kernls at compile-time. This will eliminates clBuildProgram() overhead and better kernel performance with certain driver.")
+# MESSAGE(STATUS "Build dynamic Hawaii kernels.")
+# MESSAGE(STATUS "Check OPENCL_OFFLINE_BUILD_HAWAII_KERNEL to build kernls at compile-time. This will eliminates clBuildProgram() overhead and better kernel performance with certain driver.")
add_definitions(-DCLBLAS_HAWAII_DYNAMIC_KERNEL)
-else()
- MESSAGE(STATUS "Build static Hawaii kernels.")
- MESSAGE(STATUS "Uncheck OCL_OFFLINE_BUILD_HAWAII_KERNEL to build kernls at run-time")
- MESSAGE(STATUS "Please ensure the presence of Hawaii device in the system. With certain driver/compiler flags, this might result in compile-time error.")
-endif( )
+#else()
+# MESSAGE(STATUS "Build static Hawaii kernels.")
+# MESSAGE(STATUS "Uncheck OPENCL_OFFLINE_BUILD_HAWAII_KERNEL to build kernls at run-time")
+# MESSAGE(STATUS "Please ensure the presence of Hawaii device in the system. With certain driver/compiler flags, this might result in compile-time error.")
+#endif( )
-if( NOT OCL_OFFLINE_BUILD_BONAIRE_KERNEL )
+#if( NOT OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL )
#use dynamic generated kernels
- MESSAGE(STATUS "Build dynamic Bonaire kernels.")
- MESSAGE(STATUS "Check OCL_OFFLINE_BUILD_BONAIRE_KERNEL to build kernls at compile-time. This will eliminates clBuildProgram() overhead and better kernel performance with certain driver.")
+# MESSAGE(STATUS "Build dynamic Bonaire kernels.")
+# MESSAGE(STATUS "Check OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL to build kernls at compile-time. This will eliminates clBuildProgram() overhead and better kernel performance with certain driver.")
add_definitions(-DCLBLAS_BONAIRE_DYNAMIC_KERNEL)
-else()
- MESSAGE(STATUS "Build static Bonaire kernels.")
- MESSAGE(STATUS "Uncheck OCL_OFFLINE_BUILD_BONAIRE_KERNEL to build kernls at run-time")
- MESSAGE(STATUS "Please ensure the presence of Bonaire device in the system. With certain driver/compiler flags, this might result in compile-time error.")
-endif( )
+#else()
+# MESSAGE(STATUS "Build static Bonaire kernels.")
+# MESSAGE(STATUS "Uncheck OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL to build kernls at run-time")
+# MESSAGE(STATUS "Please ensure the presence of Bonaire device in the system. With certain driver/compiler flags, this might result in compile-time error.")
+#endif( )
-if( NOT OCL_OFFLINE_BUILD_TAHITI_KERNEL )
+#if( NOT OPENCL_OFFLINE_BUILD_TAHITI_KERNEL )
#use dynamic generated kernels
- MESSAGE(STATUS "Build dynamic Tahiti kernels.")
- MESSAGE(STATUS "Check OCL_OFFLINE_BUILD_TAHITI_KERNEL to build kernls at compile-time. This will eliminates clBuildProgram() overhead and better kernel performance with certain driver.")
+# MESSAGE(STATUS "Build dynamic Tahiti kernels.")
+# MESSAGE(STATUS "Check OPENCL_OFFLINE_BUILD_TAHITI_KERNEL to build kernls at compile-time. This will eliminates clBuildProgram() overhead and better kernel performance with certain driver.")
add_definitions(-DCLBLAS_TAHITI_DYNAMIC_KERNEL)
-else( )
- MESSAGE(STATUS "Build static Tahiti kernels.")
- MESSAGE(STATUS "Uncheck OCL_OFFLINE_BUILD_TAHITI_KERNEL to build kernls at run-time")
- MESSAGE(STATUS "Please ensure the presence of Tahiti device in the system. With certain driver/compiler flags, this might result in compile-time error.")
-endif( )
+#else( )
+# MESSAGE(STATUS "Build static Tahiti kernels.")
+# MESSAGE(STATUS "Uncheck OPENCL_OFFLINE_BUILD_TAHITI_KERNEL to build kernls at run-time")
+# MESSAGE(STATUS "Please ensure the presence of Tahiti device in the system. With certain driver/compiler flags, this might result in compile-time error.")
+#endif( )
# Ask the user to verify compiler version. If OpenCL 2.0 is supported. Certain public flags can be user
-set( OCL_VERSION "1.2" CACHE STRING "The version of OpenCL supported by your driver/device" )
-set_property( CACHE OCL_VERSION PROPERTY STRINGS 2.0 1.2 1.1 )
-message( STATUS "You have confirmed OpenCL ${OCL_VERSION} is supported in your system" )
+set( OPENCL_VERSION "1.2" CACHE STRING "The version of OpenCL supported by your driver/device" )
+set_property( CACHE OPENCL_VERSION PROPERTY STRINGS 2.0 1.2 1.1 )
+message( STATUS "You have confirmed OpenCL ${OPENCL_VERSION} is supported in your system" )
# By default test-correctness is linked and tested against ACML library.
# However, test-correctness can instead use NETLIB as a reference library
@@ -172,16 +175,16 @@ if( UNIX )
endif()
# set the path to specific OpenCL compiler
-set( OCL_COMPILER_DIR "OPENCL COMPILER PATH" CACHE PATH "OPENCL COMPILER PATH")
-if ( ${OCL_COMPILER_DIR} STREQUAL "OPENCL COMPILER PATH")
- message( STATUS "Use default OpenCL Compiler")
+set( OPENCL_COMPILER_DIR "OPENCL COMPILER PATH" CACHE PATH "OPENCL COMPILER PATH")
+if ( ${OPENCL_COMPILER_DIR} STREQUAL "OPENCL COMPILER PATH")
+ message( STATUS "Using default OpenCL Compiler")
set(ENV_PATH "$ENV{PATH}")
else ()
- message( STATUS "OPENCL COMPILER: ${OCL_COMPILER_DIR}")
+ message( STATUS "OPENCL COMPILER: ${OPENCL_COMPILER_DIR}")
if(UNIX)
- set(ENV_PATH "${OCL_COMPILER_DIR}")
+ set(ENV_PATH "${OPENCL_COMPILER_DIR}")
else()
- set(ENV_PATH "${OCL_COMPILER_DIR}")
+ set(ENV_PATH "${OPENCL_COMPILER_DIR}")
endif()
endif()
diff --git a/src/client/clfunc_common.hpp b/src/client/clfunc_common.hpp
index 01363a9..3a66d61 100644
--- a/src/client/clfunc_common.hpp
+++ b/src/client/clfunc_common.hpp
@@ -246,8 +246,9 @@ public:
props_[2] = 0;
ctx_ = clCreateContext(props_, 1, &device_, NULL, NULL, &err);
OPENCL_V_THROW(err, "creating context");
- queue_ = clCreateCommandQueue(ctx_, device_, 0, &err);
-
+ for (unsigned int i = 0; i < numQueues; i++) {
+ queues_[i] = clCreateCommandQueue(ctx_, device_, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err);
+ }
timer_id = timer.getUniqueID( "clfunc", 0 );
@@ -258,7 +259,9 @@ public:
err = clblasSetup();
if (err != CL_SUCCESS) {
std::cerr << "clblasSetup() failed with %d\n";
- clReleaseCommandQueue(queue_);
+ for (unsigned int i = 0; i < numQueues; i++) {
+ clReleaseCommandQueue(queues_[i]);
+ }
clReleaseContext(ctx_);
}
}
@@ -266,8 +269,10 @@ public:
virtual ~clblasFunc()
{
clblasTeardown();
- OPENCL_V_THROW( clReleaseCommandQueue(queue_),
- "releasing command queue" );
+
+ for (unsigned int i = 0; i < numQueues; i++) {
+ OPENCL_V_THROW( clReleaseCommandQueue(queues_[i]), "releasing command queue" );
+ }
OPENCL_V_THROW( clReleaseContext(ctx_), "releasing context" );
}
@@ -337,7 +342,8 @@ protected:
cl_device_id device_;
cl_context_properties props_[3];
cl_context ctx_;
- cl_command_queue queue_;
+ static const unsigned int numQueues = 4;
+ cl_command_queue queues_[numQueues];
clblasOrder order_;
cl_event event_;
size_t maxMemAllocSize;
diff --git a/src/client/clfunc_xgemm.hpp b/src/client/clfunc_xgemm.hpp
index b676f13..8efaf63 100644
--- a/src/client/clfunc_xgemm.hpp
+++ b/src/client/clfunc_xgemm.hpp
@@ -55,8 +55,9 @@ template <typename T>
class xGemm : public clblasFunc
{
public:
- xGemm(StatisticalTimer& timer, cl_device_type devType) :
- clblasFunc(timer, devType)
+ xGemm(StatisticalTimer& timer, cl_device_type devType, unsigned int iNumQueuesToUse) :
+ clblasFunc(timer, devType),
+ numQueuesToUse(iNumQueuesToUse)
{
timer.getUniqueID("clGemm", 0);
}
@@ -376,19 +377,19 @@ public:
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -400,7 +401,7 @@ public:
void reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -410,7 +411,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T), buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
buffer_.c_, 0, NULL, NULL);
@@ -434,25 +435,25 @@ public:
(buffer_.ldc_ * buffer_.c_num_vectors_ +
buffer_.offC_) * sizeof(T),
NULL, &err);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
buffer_.c_, 0, NULL, NULL);
xGemm_Function(false);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T), buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
buffer_.c_, 0, NULL, &event_);
@@ -503,28 +504,28 @@ public:
buffer_.offC_) * sizeof(T),
NULL, &err);
/*
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
buffer_.c_, 0, NULL, NULL);*/
- err = clEnqueueWriteBufferRect(queue_, buffer_.buf_a_, CL_TRUE, a_buffer_origin, a_host_origin, a_region, a_buffer_row_pitch,
+ err = clEnqueueWriteBufferRect(queues_[0], buffer_.buf_a_, CL_TRUE, a_buffer_origin, a_host_origin, a_region, a_buffer_row_pitch,
a_buffer_slice_pitch, a_host_row_pitch, a_host_slice_pitch, buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBufferRect(queue_, buffer_.buf_b_, CL_TRUE, b_buffer_origin, b_host_origin, b_region, b_buffer_row_pitch,
+ err = clEnqueueWriteBufferRect(queues_[0], buffer_.buf_b_, CL_TRUE, b_buffer_origin, b_host_origin, b_region, b_buffer_row_pitch,
b_buffer_slice_pitch, b_host_row_pitch, b_host_slice_pitch, buffer_.b_, 0, NULL, NULL);
- err = clEnqueueWriteBufferRect(queue_, buffer_.buf_c_, CL_TRUE, c_buffer_origin, c_host_origin, c_region, c_buffer_row_pitch,
+ err = clEnqueueWriteBufferRect(queues_[0], buffer_.buf_c_, CL_TRUE, c_buffer_origin, c_host_origin, c_region, c_buffer_row_pitch,
c_buffer_slice_pitch, c_host_row_pitch, c_host_slice_pitch, buffer_.c_, 0, NULL, NULL);
if(buffer_.trans_a_==clblasNoTrans)
@@ -546,12 +547,12 @@ public:
buffer_.ldc_=buffer_.m_;
xGemm_Function(false);
/*
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T), buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
buffer_.c_, 0, NULL, &event_);
*/
- err = ::clEnqueueReadBufferRect(queue_, buffer_.buf_c_, CL_TRUE, c_buffer_origin, c_host_origin, c_region, c_buffer_row_pitch,
+ err = ::clEnqueueReadBufferRect(queues_[0], buffer_.buf_c_, CL_TRUE, c_buffer_origin, c_host_origin, c_region, c_buffer_row_pitch,
c_buffer_slice_pitch, c_host_row_pitch, c_host_slice_pitch, buffer_.c_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -579,15 +580,15 @@ public:
// map the buffers to pointers at host device
T *map_a,*map_b,*map_c;
- map_a = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_a = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.lda_*buffer_.a_num_vectors_ +
buffer_.offA_) * sizeof(T),
0, NULL, NULL, &err);
- map_b = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_b = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.ldb_*buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
- map_c = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_c_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_c = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_c_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.lda_*buffer_.c_num_vectors_ +
buffer_.offC_) * sizeof(T),
0, NULL, NULL, &err);
@@ -596,18 +597,18 @@ public:
memcpy( map_b, buffer_.b_, ( buffer_.ldb_*buffer_.b_num_vectors_ + buffer_.offB_) * sizeof( T ) );
memcpy( map_c, buffer_.c_, ( buffer_.ldc_*buffer_.c_num_vectors_ + buffer_.offC_) * sizeof( T ) );
// unmap the buffers
- clEnqueueUnmapMemObject(queue_, buffer_.buf_a_, map_a, 0, NULL, NULL);
- clEnqueueUnmapMemObject(queue_, buffer_.buf_b_, map_b, 0, NULL, NULL);
- clEnqueueUnmapMemObject(queue_, buffer_.buf_c_, map_c, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_a_, map_a, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_b_, map_b, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_c_, map_c, 0, NULL, NULL);
// calling clBLAS
xGemm_Function(false);
// map the C buffer again to read output
- map_c = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_c_, CL_TRUE, CL_MAP_READ, 0,
+ map_c = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_c_, CL_TRUE, CL_MAP_READ, 0,
(buffer_.lda_*buffer_.c_num_vectors_ +
buffer_.offC_) * sizeof(T),
0, NULL, NULL, &err);
memcpy( map_c, buffer_.c_, ( buffer_.ldc_*buffer_.c_num_vectors_ + buffer_.offC_) * sizeof( T ) );
- clEnqueueUnmapMemObject(queue_, buffer_.buf_c_, map_c, 0, NULL, &event_);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_c_, map_c, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -652,7 +653,7 @@ public:
buffer_.offC_) * sizeof(T),
buffer_.c_, &err);
xGemm_Function(false);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T), buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
buffer_.c_, 0, NULL, &event_);
@@ -683,15 +684,15 @@ public:
// map the buffers to pointers at host devices
T *map_a,*map_b,*map_c;
- map_a = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_a = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.lda_*buffer_.a_num_vectors_ +
buffer_.offA_) * sizeof(T),
0, NULL, NULL, &err);
- map_b = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_b = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.ldb_*buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
- map_c = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_c_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_c = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_c_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.lda_*buffer_.c_num_vectors_ +
buffer_.offC_) * sizeof(T),
0, NULL, NULL, &err);
@@ -700,18 +701,18 @@ public:
memcpy( map_b, buffer_.b_, ( buffer_.ldb_*buffer_.b_num_vectors_ + buffer_.offB_) * sizeof( T ) );
memcpy( map_c, buffer_.c_, ( buffer_.ldc_*buffer_.c_num_vectors_ + buffer_.offC_) * sizeof( T ) );
// unmap the buffers
- clEnqueueUnmapMemObject(queue_, buffer_.buf_a_, map_a, 0, NULL, NULL);
- clEnqueueUnmapMemObject(queue_, buffer_.buf_b_, map_b, 0, NULL, NULL);
- clEnqueueUnmapMemObject(queue_, buffer_.buf_c_, map_c, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_a_, map_a, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_b_, map_b, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_c_, map_c, 0, NULL, NULL);
// calling clBLAS
xGemm_Function(false);
// map the C buffer again to read output
- map_c = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_c_, CL_TRUE, CL_MAP_READ, 0,
+ map_c = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_c_, CL_TRUE, CL_MAP_READ, 0,
(buffer_.lda_*buffer_.c_num_vectors_ +
buffer_.offC_) * sizeof(T),
0, NULL, NULL, &err);
memcpy( map_c, buffer_.c_, ( buffer_.ldc_*buffer_.c_num_vectors_ + buffer_.offC_) * sizeof( T ) );
- clEnqueueUnmapMemObject(queue_, buffer_.buf_c_, map_c, 0, NULL, &event_);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_c_, map_c, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -982,7 +983,8 @@ protected:
private:
xGemmBuffer<T> buffer_;
void xGemm_Function(bool flush, cl_uint apiCallCount = 1);
-
+ unsigned int numQueuesToUse;
+ cl_event events_[numQueues];
}; // class xgemm
@@ -991,20 +993,43 @@ void
xGemm<cl_float>::
xGemm_Function(bool flush, cl_uint apiCallCount )
{
- for (int i = 0; i < apiCallCount; i++)
+ for (unsigned int i = 0; i < numQueues; i++) {
+ events_[i] = NULL;
+ }
+ for (unsigned int i = 0; i < apiCallCount; i++)
{
clblasSgemm(order_, buffer_.trans_a_, buffer_.trans_b_,
buffer_.m_, buffer_.n_, buffer_.k_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueuesToUse, queues_, 0, NULL, events_);
}
//flush==true if only the kernel time (library call) is timed
//flush==false if memory time is also timed
if (flush==true)
{
- clWaitForEvents(1, &event_);
+ // check if any valid events returned
+ cl_uint numValidEvents = 0;
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ if (events_[i]) {
+ cl_uint clReferenceCount;
+ cl_int err = clGetEventInfo(events_[i], CL_EVENT_REFERENCE_COUNT, sizeof(clReferenceCount), &clReferenceCount, NULL);
+ if ( err == CL_SUCCESS) {
+ //printf("events[%u/%u] has %u references\n", i, numQueuesToUse, clReferenceCount );
+ numValidEvents++;
+ } else {
+ //printf("events[%u/%u] invalid; err = %i\n", i, numQueuesToUse, err );
+ }
+ } else {
+ //printf("events[%u/%u] is NULL\n", i, numQueuesToUse );
+ }
+ }
+
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ clFlush(queues_[i]);
+ }
+ clWaitForEvents(numValidEvents, events_);
}
}
@@ -1013,20 +1038,43 @@ void
xGemm<cl_double>::
xGemm_Function(bool flush, cl_uint apiCallCount )
{
- for (int i = 0; i < apiCallCount; i++)
+ for (unsigned int i = 0; i < numQueues; i++) {
+ events_[i] = NULL;
+ }
+ for (unsigned int i = 0; i < apiCallCount; i++)
{
clblasDgemm(order_, buffer_.trans_a_, buffer_.trans_b_,
buffer_.m_, buffer_.n_, buffer_.k_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueuesToUse, queues_, 0, NULL, events_);
}
//flush==true if only the kernel time (library call) is timed
//flush==false if memory time is also timed
if (flush==true)
{
- clWaitForEvents(1, &event_);
+ // check if any valid events returned
+ cl_uint numValidEvents = 0;
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ if (events_[i]) {
+ cl_uint clReferenceCount;
+ cl_int err = clGetEventInfo(events_[i], CL_EVENT_REFERENCE_COUNT, sizeof(clReferenceCount), &clReferenceCount, NULL);
+ if ( err == CL_SUCCESS) {
+ //printf("events[%u/%u] has %u references\n", i, numQueuesToUse, clReferenceCount );
+ numValidEvents++;
+ } else {
+ //printf("events[%u/%u] invalid; err = %i\n", i, numQueuesToUse, err );
+ }
+ } else {
+ //printf("events[%u/%u] is NULL\n", i, numQueuesToUse );
+ }
+ }
+
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ clFlush(queues_[i]);
+ }
+ clWaitForEvents(numValidEvents, events_);
}
}
@@ -1035,20 +1083,43 @@ void
xGemm<cl_float2>::
xGemm_Function(bool flush, cl_uint apiCallCount )
{
- for (int i = 0; i < apiCallCount; i++)
+ for (unsigned int i = 0; i < numQueues; i++) {
+ events_[i] = NULL;
+ }
+ for (unsigned int i = 0; i < apiCallCount; i++)
{
clblasCgemm(order_, buffer_.trans_a_, buffer_.trans_b_,
buffer_.m_, buffer_.n_, buffer_.k_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueuesToUse, queues_, 0, NULL, events_);
}
//flush==true if only the kernel time (library call) is timed
//flush==false if memory time is also timed
if (flush==true)
{
- clWaitForEvents(1, &event_);
+ // check if any valid events returned
+ cl_uint numValidEvents = 0;
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ if (events_[i]) {
+ cl_uint clReferenceCount;
+ cl_int err = clGetEventInfo(events_[i], CL_EVENT_REFERENCE_COUNT, sizeof(clReferenceCount), &clReferenceCount, NULL);
+ if ( err == CL_SUCCESS) {
+ //printf("events[%u/%u] has %u references\n", i, numQueuesToUse, clReferenceCount );
+ numValidEvents++;
+ } else {
+ //printf("events[%u/%u] invalid; err = %i\n", i, numQueuesToUse, err );
+ }
+ } else {
+ //printf("events[%u/%u] is NULL\n", i, numQueuesToUse );
+ }
+ }
+
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ clFlush(queues_[i]);
+ }
+ clWaitForEvents(numValidEvents, events_);
}
}
@@ -1057,20 +1128,43 @@ void
xGemm<cl_double2>::
xGemm_Function(bool flush, cl_uint apiCallCount )
{
- for (int i = 0; i < apiCallCount; i++)
+ for (unsigned int i = 0; i < numQueues; i++) {
+ events_[i] = NULL;
+ }
+ for (unsigned int i = 0; i < apiCallCount; i++)
{
clblasZgemm(order_, buffer_.trans_a_, buffer_.trans_b_,
buffer_.m_, buffer_.n_, buffer_.k_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueuesToUse, queues_, 0, NULL, events_);
}
//flush==true if only the kernel time (library call) is timed
//flush==false if memory time is also timed
if (flush==true)
{
- clWaitForEvents(1, &event_);
+ // check if any valid events returned
+ cl_uint numValidEvents = 0;
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ if (events_[i]) {
+ cl_uint clReferenceCount;
+ cl_int err = clGetEventInfo(events_[i], CL_EVENT_REFERENCE_COUNT, sizeof(clReferenceCount), &clReferenceCount, NULL);
+ if ( err == CL_SUCCESS) {
+ //printf("events[%u/%u] has %u references\n", i, numQueuesToUse, clReferenceCount );
+ numValidEvents++;
+ } else {
+ //printf("events[%u/%u] invalid; err = %i\n", i, numQueuesToUse, err );
+ }
+ } else {
+ //printf("events[%u/%u] is NULL\n", i, numQueuesToUse );
+ }
+ }
+ for (unsigned int i = 0; i < numQueuesToUse; i++) {
+ clFlush(queues_[i]);
+ }
+
+ clWaitForEvents(numValidEvents, events_);
}
}
diff --git a/src/client/clfunc_xgemv.hpp b/src/client/clfunc_xgemv.hpp
index cc85109..d7c0761 100644
--- a/src/client/clfunc_xgemv.hpp
+++ b/src/client/clfunc_xgemv.hpp
@@ -227,7 +227,7 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
@@ -235,21 +235,21 @@ public:
if (buffer_.trans_a_ == clblasNoTrans)
{
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_x_, CL_TRUE, 0,
buffer_.n_*sizeof(T),
buffer_.x_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_y_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_y_, CL_TRUE, 0,
buffer_.m_*sizeof(T),
buffer_.y_, 0, NULL, NULL);
}
else
{
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_x_, CL_TRUE, 0,
buffer_.m_*sizeof(T),
buffer_.x_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_y_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_y_, CL_TRUE, 0,
buffer_.n_*sizeof(T),
buffer_.y_, 0, NULL, NULL);
}
@@ -261,13 +261,13 @@ public:
if (buffer_.trans_a_ == clblasNoTrans)
{
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_y_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_y_, CL_TRUE, 0,
buffer_.m_*sizeof(T),
buffer_.y_, 0, NULL, NULL);
}
else
{
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_y_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_y_, CL_TRUE, 0,
buffer_.n_*sizeof(T),
buffer_.y_, 0, NULL, NULL);
}
@@ -316,7 +316,7 @@ call_func()
clblasSgemv(order_, buffer_.trans_a_, buffer_.m_, buffer_.n_,
buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.beta_,
- buffer_.buf_y_, 0, 1, 1, &queue_, 0, NULL, &event_);
+ buffer_.buf_y_, 0, 1, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -332,7 +332,7 @@ call_func()
clblasDgemv(order_, buffer_.trans_a_, buffer_.m_, buffer_.n_,
buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.beta_,
- buffer_.buf_y_, 0, 1, 1, &queue_, 0, NULL, &event_);
+ buffer_.buf_y_, 0, 1, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -348,7 +348,7 @@ call_func()
clblasCgemv(order_, buffer_.trans_a_, buffer_.m_, buffer_.n_,
buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.beta_,
- buffer_.buf_y_, 0, 1, 1, &queue_, 0, NULL, &event_);
+ buffer_.buf_y_, 0, 1, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -364,7 +364,7 @@ call_func()
clblasZgemv(order_, buffer_.trans_a_, buffer_.m_, buffer_.n_,
buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.beta_,
- buffer_.buf_y_, 0, 1, 1, &queue_, 0, NULL, &event_);
+ buffer_.buf_y_, 0, 1, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
diff --git a/src/client/clfunc_xger.hpp b/src/client/clfunc_xger.hpp
index d2f36db..d1831a3 100644
--- a/src/client/clfunc_xger.hpp
+++ b/src/client/clfunc_xger.hpp
@@ -182,14 +182,14 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.a_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.a_, CL_TRUE, 0,
buffer_.lda_*buffer_.a_num_vectors_*sizeof(T),
buffer_.A, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.x_, CL_TRUE, 0,
buffer_.m_*sizeof(T),
buffer_.X, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.y_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.y_, CL_TRUE, 0,
buffer_.n_*sizeof(T),
buffer_.Y, 0, NULL, NULL);
}
@@ -197,7 +197,7 @@ public:
void reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.x_, CL_TRUE, 0,
buffer_.m_,
buffer_.x_, 0, NULL, NULL);
}
@@ -257,7 +257,7 @@ call_func()
{
timer.Start(timer_id);
clblasSger(buffer_.order_, buffer_.m_, buffer_.n_, buffer_.alpha, buffer_.x_, buffer_.offX, 1, buffer_.y_, buffer_.offY,
- 1, buffer_.a_, buffer_.offA, buffer_.lda_, 1, &queue_, 0, NULL,
+ 1, buffer_.a_, buffer_.offA, buffer_.lda_, numQueues, queues_, 0, NULL,
&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -270,7 +270,7 @@ call_func()
{
timer.Start(timer_id);
clblasDger(buffer_.order_, buffer_.m_, buffer_.n_, buffer_.alpha, buffer_.x_, buffer_.offX, 1, buffer_.y_, buffer_.offY,
- 1, buffer_.a_, buffer_.offA, buffer_.lda_, 1, &queue_, 0, NULL,
+ 1, buffer_.a_, buffer_.offA, buffer_.lda_, numQueues, queues_, 0, NULL,
&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -283,7 +283,7 @@ call_func()
//{
// timer.Start(timer_id);
// clblasCger(order_, buffer_.m_, buffer_.n, buffer_a_, 0,
-// buffer_.lda_, buffer_x_, 0, 1, 1, &queue_, 0, NULL,
+// buffer_.lda_, buffer_x_, 0, 1, numQueues, queues_, 0, NULL,
// &event_);
// clWaitForEvents(1, &event_);
// timer.Stop(timer_id);
@@ -297,7 +297,7 @@ call_func()
// timer.Start(timer_id);
// clblasZger(order_, buffer_.uplo_, buffer_.trans_a_,
// buffer_.diag_, buffer_.m_, buffer_a_, 0,
-// buffer_.lda_, buffer_x_, 0, 1, 1, &queue_, 0, NULL,
+// buffer_.lda_, buffer_x_, 0, 1, numQueues, queues_, 0, NULL,
// &event_);
// clWaitForEvents(1, &event_);
// timer.Stop(timer_id);
diff --git a/src/client/clfunc_xgerc.hpp b/src/client/clfunc_xgerc.hpp
index ed39f79..a8e780a 100644
--- a/src/client/clfunc_xgerc.hpp
+++ b/src/client/clfunc_xgerc.hpp
@@ -260,15 +260,15 @@ void xGerc<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.X, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.X, CL_TRUE, 0,
buffer.M*sizeof(T),
buffer.cpuX, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.Y, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.Y, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuY, 0, NULL, NULL);
}
@@ -277,7 +277,7 @@ template <typename T>
void xGerc<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);;
@@ -288,7 +288,7 @@ void xGerc<cl_float2>::call_func()
{
timer.Start(timer_id);
clblasCgerc(buffer.order, buffer.M, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -298,7 +298,7 @@ void xGerc<cl_double2>::call_func()
{
timer.Start(timer_id);
clblasZgerc(buffer.order, buffer.M, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xgeru.hpp b/src/client/clfunc_xgeru.hpp
index dbcecc9..fc62e24 100644
--- a/src/client/clfunc_xgeru.hpp
+++ b/src/client/clfunc_xgeru.hpp
@@ -195,15 +195,15 @@ void xGeru<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.X, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.X, CL_TRUE, 0,
buffer.M*sizeof(T),
buffer.cpuX, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.Y, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.Y, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuY, 0, NULL, NULL);
}
@@ -212,7 +212,7 @@ template <typename T>
void xGeru<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);;
@@ -223,7 +223,7 @@ void xGeru<cl_float2>::call_func()
{
timer.Start(timer_id);
clblasCgeru(buffer.order, buffer.M, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -233,7 +233,7 @@ void xGeru<cl_double2>::call_func()
{
timer.Start(timer_id);
clblasZgeru(buffer.order, buffer.M, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xhemm.hpp b/src/client/clfunc_xhemm.hpp
index 9f4047e..08f6ba2 100644
--- a/src/client/clfunc_xhemm.hpp
+++ b/src/client/clfunc_xhemm.hpp
@@ -117,7 +117,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(T),
buffer.ldc*buffer.N*sizeof(T),
buffer.cpuC,0,NULL,NULL);
@@ -470,16 +470,16 @@ void xHemm<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.a_num_vectors * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE,
buffer.offb * sizeof(T),
buffer.ldb*buffer.N*sizeof(T),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(T),
buffer.ldc*buffer.N*sizeof(T),
buffer.cpuC, 0, NULL, NULL);
@@ -489,7 +489,7 @@ template <typename T>
void xHemm<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE, 0,
buffer.ldc*buffer.N*sizeof(T),
buffer.cpuC, 0, NULL, NULL);
}
@@ -500,7 +500,7 @@ void xHemm<cl_float2>::call_func()
timer.Start(timer_id);
clblasChemm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -523,26 +523,26 @@ void xHemm<cl_float2>::roundtrip_func()
buffer.N*buffer.ldc*sizeof(cl_float2),
NULL, &err);
//write gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(cl_float2),
buffer.a_num_vectors * buffer.lda*sizeof(cl_float2),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE,
buffer.offb * sizeof(cl_float2),
buffer.ldb*buffer.N*sizeof(cl_float2),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_float2),
buffer.ldc*buffer.N*sizeof(cl_float2),
buffer.cpuC, 0, NULL, NULL);
clblasChemm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_float2),
buffer.ldc*buffer.N*sizeof(cl_float2),
buffer.cpuC, 0, NULL, &event_);
@@ -556,7 +556,7 @@ void xHemm<cl_double2>::call_func()
timer.Start(timer_id);
clblasZhemm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -578,26 +578,26 @@ void xHemm<cl_double2>::roundtrip_func()
buffer.N*buffer.ldc*sizeof(cl_double2),
NULL, &err);
//write gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(cl_double2),
buffer.a_num_vectors * buffer.lda*sizeof(cl_double2),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE,
buffer.offb * sizeof(cl_double2),
buffer.ldb*buffer.N*sizeof(cl_double2),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_double2),
buffer.ldc*buffer.N*sizeof(cl_double2),
buffer.cpuC, 0, NULL, NULL);
clblasZhemm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_double2),
buffer.ldc*buffer.N*sizeof(cl_double2),
buffer.cpuC, 0, NULL, &event_);
diff --git a/src/client/clfunc_xhemv.hpp b/src/client/clfunc_xhemv.hpp
index 6211114..a09a6f6 100644
--- a/src/client/clfunc_xhemv.hpp
+++ b/src/client/clfunc_xhemv.hpp
@@ -200,15 +200,15 @@ void xHemv<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.X, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.X, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuX, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.Y, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.Y, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuY, 0, NULL, NULL);
}
@@ -217,7 +217,7 @@ template <typename T>
void xHemv<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);;
@@ -229,7 +229,7 @@ void xHemv<cl_float2>::call_func()
timer.Start(timer_id);
clblasChemv(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.A,
buffer.offa, buffer.lda, buffer.X, buffer.offx, buffer.incx,
- buffer.beta, buffer.Y, buffer.offy, buffer.incy, 1, &queue_, 0, NULL,&event_);
+ buffer.beta, buffer.Y, buffer.offy, buffer.incy, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -240,7 +240,7 @@ void xHemv<cl_double2>::call_func()
timer.Start(timer_id);
clblasZhemv(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.A,
buffer.offa, buffer.lda, buffer.X, buffer.offx, buffer.incx,
- buffer.beta, buffer.Y, buffer.offy, buffer.incy, 1, &queue_, 0, NULL,&event_);
+ buffer.beta, buffer.Y, buffer.offy, buffer.incy, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xher.hpp b/src/client/clfunc_xher.hpp
index 5144b22..fec0953 100644
--- a/src/client/clfunc_xher.hpp
+++ b/src/client/clfunc_xher.hpp
@@ -235,12 +235,12 @@ void xHer<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.X, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.X, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuX, 0, NULL, NULL);
}
@@ -249,7 +249,7 @@ template <typename T>
void xHer<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);;
@@ -260,7 +260,7 @@ void xHer<cl_float2>::call_func()
{
timer.Start(timer_id);
clblasCher(buffer.order, buffer.uplo, buffer.N, buffer.alpha.s[0], buffer.X, buffer.offx,
- buffer.incx, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -270,7 +270,7 @@ void xHer<cl_double2>::call_func()
{
timer.Start(timer_id);
clblasZher(buffer.order, buffer.uplo, buffer.N, buffer.alpha.s[0], buffer.X, buffer.offx,
- buffer.incx, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xher2.hpp b/src/client/clfunc_xher2.hpp
index aec7cc8..dd7613f 100644
--- a/src/client/clfunc_xher2.hpp
+++ b/src/client/clfunc_xher2.hpp
@@ -255,15 +255,15 @@ void xHer2<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.X, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.X, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuX, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.Y, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.Y, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuY, 0, NULL, NULL);
}
@@ -272,7 +272,7 @@ template <typename T>
void xHer2<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);;
@@ -284,7 +284,7 @@ void xHer2<cl_float2>::call_func()
timer.Start(timer_id);
clblasCher2(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.X, buffer.offx,
buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa,
- buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -295,7 +295,7 @@ void xHer2<cl_double2>::call_func()
timer.Start(timer_id);
clblasZher2(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.X, buffer.offx,
buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa,
- buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xher2k.hpp b/src/client/clfunc_xher2k.hpp
index 15095fa..272dc2c 100644
--- a/src/client/clfunc_xher2k.hpp
+++ b/src/client/clfunc_xher2k.hpp
@@ -313,13 +313,13 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.A_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.A_, CL_TRUE,
buffer_.offa_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.cpuA_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offa_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -329,7 +329,7 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -339,7 +339,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_*sizeof(T), buffer_.ldc_*buffer_.c_num_vectors_*sizeof(T),
buffer_.cpuC_, 0, NULL, NULL);
}
@@ -581,7 +581,7 @@ xHer2k<cl_float2>::call_func()
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.B_, buffer_.offb_, buffer_.ldb_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -610,9 +610,9 @@ xHer2k<cl_float2>::roundtrip_func()
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.B_, buffer_.offb_, buffer_.ldb_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_ * sizeof(cl_float2),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(cl_float2),
@@ -632,7 +632,7 @@ xHer2k<cl_double2>::call_func()
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.B_, buffer_.offb_, buffer_.ldb_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -663,9 +663,9 @@ xHer2k<cl_double2>::roundtrip_func()
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.B_, buffer_.offb_, buffer_.ldb_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_ * sizeof(cl_double2),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(cl_double2),
diff --git a/src/client/clfunc_xherk.hpp b/src/client/clfunc_xherk.hpp
index 74871a3..446ac50 100644
--- a/src/client/clfunc_xherk.hpp
+++ b/src/client/clfunc_xherk.hpp
@@ -242,13 +242,13 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.A_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.A_, CL_TRUE,
buffer_.offa_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.cpuA_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offa_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -258,7 +258,7 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -268,7 +268,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_*sizeof(T), buffer_.ldc_*buffer_.c_num_vectors_*sizeof(T),
buffer_.cpuC_, 0, NULL, NULL);
}
@@ -447,7 +447,7 @@ xHerk<cl_float2>::call_func()
buffer_.N_, buffer_.K_, buffer_.alpha_.s[0],
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -474,9 +474,9 @@ xHerk<cl_float2>::roundtrip_func()
buffer_.N_, buffer_.K_, buffer_.alpha_.s[0],
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_ * sizeof(cl_float2),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(cl_float2),
@@ -495,7 +495,7 @@ xHerk<cl_double2>::call_func()
buffer_.N_, buffer_.K_, buffer_.alpha_.s[0],
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -522,9 +522,9 @@ xHerk<cl_double2>::roundtrip_func()
buffer_.N_, buffer_.K_, buffer_.alpha_.s[0],
buffer_.A_, buffer_.offa_, buffer_.lda_,
buffer_.beta_.s[0], buffer_.C_, buffer_.offc_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.C_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.C_, CL_TRUE,
buffer_.offc_ * sizeof(cl_double2),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(cl_double2),
diff --git a/src/client/clfunc_xsymm.hpp b/src/client/clfunc_xsymm.hpp
index a7558e9..2e279c2 100644
--- a/src/client/clfunc_xsymm.hpp
+++ b/src/client/clfunc_xsymm.hpp
@@ -89,7 +89,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(T), buffer.ldc * buffer.N *
sizeof(T),
buffer.cpuC, 0, NULL, NULL);
@@ -389,14 +389,14 @@ void xSymm<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.a_num_vectors * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE, 0,
buffer.ldb*buffer.N*sizeof(T),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE, 0,
buffer.ldc*buffer.N*sizeof(T),
buffer.cpuC, 0, NULL, NULL);
}
@@ -405,7 +405,7 @@ template <typename T>
void xSymm<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE, 0,
buffer.ldc*buffer.N*sizeof(T),
buffer.cpuC, 0, NULL, NULL);
}
@@ -416,7 +416,7 @@ void xSymm<cl_float>::call_func()
timer.Start(timer_id);
clblasSsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -439,23 +439,23 @@ void xSymm<cl_float>::roundtrip_func()
buffer.N*buffer.ldc*sizeof(cl_float),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(cl_float),
buffer.a_num_vectors * buffer.lda*sizeof(cl_float),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE, 0,
buffer.ldb*buffer.N*sizeof(cl_float),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE, 0,
buffer.ldc*buffer.N*sizeof(cl_float),
buffer.cpuC, 0, NULL, NULL);
//call func
clblasSsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_float), buffer.ldc * buffer.N *
sizeof(cl_float),
buffer.cpuC, 0, NULL, &event_);
@@ -469,7 +469,7 @@ void xSymm<cl_double>::call_func()
timer.Start(timer_id);
clblasDsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -492,23 +492,23 @@ void xSymm<cl_double>::roundtrip_func()
buffer.N*buffer.ldc*sizeof(cl_double),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(cl_double),
buffer.a_num_vectors * buffer.lda*sizeof(cl_double),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE, 0,
buffer.ldb*buffer.N*sizeof(cl_double),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE, 0,
buffer.ldc*buffer.N*sizeof(cl_double),
buffer.cpuC, 0, NULL, NULL);
//call func
clblasDsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_double), buffer.ldc * buffer.N *
sizeof(cl_double),
buffer.cpuC, 0, NULL, &event_);
@@ -522,7 +522,7 @@ void xSymm<cl_float2>::call_func()
timer.Start(timer_id);
clblasCsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -545,23 +545,23 @@ void xSymm<cl_float2>::roundtrip_func()
buffer.N*buffer.ldc*sizeof(cl_float2),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(cl_float2),
buffer.a_num_vectors * buffer.lda*sizeof(cl_float2),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE, 0,
buffer.ldb*buffer.N*sizeof(cl_float2),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE, 0,
buffer.ldc*buffer.N*sizeof(cl_float2),
buffer.cpuC, 0, NULL, NULL);
//call func
clblasCsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_float2), buffer.ldc * buffer.N *
sizeof(cl_float2),
buffer.cpuC, 0, NULL, &event_);
@@ -575,7 +575,7 @@ void xSymm<cl_double2>::call_func()
timer.Start(timer_id);
clblasZsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -598,23 +598,23 @@ void xSymm<cl_double2>::roundtrip_func()
buffer.N*buffer.ldc*sizeof(cl_double2),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(cl_double2),
buffer.a_num_vectors * buffer.lda*sizeof(cl_double2),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.B, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.B, CL_TRUE, 0,
buffer.ldb*buffer.N*sizeof(cl_double2),
buffer.cpuB, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.C, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.C, CL_TRUE, 0,
buffer.ldc*buffer.N*sizeof(cl_double2),
buffer.cpuC, 0, NULL, NULL);
//call func
clblasZsymm(buffer.order, buffer.side, buffer.uplo, buffer.M, buffer.N,
buffer.alpha, buffer.A, buffer.offa, buffer.lda, buffer.B, buffer.offb,
- buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, 1, &queue_,
+ buffer.ldb, buffer.beta, buffer.C, buffer.offc, buffer.ldc, numQueues, queues_,
0, NULL,NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer.C, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer.C, CL_TRUE,
buffer.offc * sizeof(cl_double2), buffer.ldc * buffer.N *
sizeof(cl_double2),
buffer.cpuC, 0, NULL, &event_);
diff --git a/src/client/clfunc_xsymv.hpp b/src/client/clfunc_xsymv.hpp
index c928541..fd2f8f4 100644
--- a/src/client/clfunc_xsymv.hpp
+++ b/src/client/clfunc_xsymv.hpp
@@ -172,17 +172,17 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_x_, CL_TRUE, 0,
buffer_.n_*sizeof(T),
buffer_.x_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_y_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_y_, CL_TRUE, 0,
buffer_.n_*sizeof(T),
buffer_.y_, 0, NULL, NULL);
}
@@ -191,7 +191,7 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_y_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_y_, CL_TRUE, 0,
buffer_.n_*sizeof(T),
buffer_.y_, 0, NULL, NULL);
}
@@ -237,7 +237,7 @@ call_func()
clblasSsymv(order_, buffer_.uplo_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_x_, 0, 1, buffer_.beta_, buffer_.buf_y_,
- 0, 1, 1, &queue_, 0, NULL, &event_);
+ 0, 1, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -253,7 +253,7 @@ call_func()
clblasDsymv(order_, buffer_.uplo_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_x_, 0, 1, buffer_.beta_, buffer_.buf_y_,
- 0, 1, 1, &queue_, 0, NULL, &event_);
+ 0, 1, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
diff --git a/src/client/clfunc_xsyr.hpp b/src/client/clfunc_xsyr.hpp
index 4c70e69..896b941 100644
--- a/src/client/clfunc_xsyr.hpp
+++ b/src/client/clfunc_xsyr.hpp
@@ -187,12 +187,12 @@ void xSyr<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.X, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.X, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuX, 0, NULL, NULL);
}
@@ -201,7 +201,7 @@ template <typename T>
void xSyr<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);;
@@ -212,7 +212,7 @@ void xSyr<cl_float>::call_func()
{
timer.Start(timer_id);
clblasSsyr(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -222,7 +222,7 @@ void xSyr<cl_double>::call_func()
{
timer.Start(timer_id);
clblasSsyr(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xsyr2.hpp b/src/client/clfunc_xsyr2.hpp
index 9977d08..08b0b9b 100644
--- a/src/client/clfunc_xsyr2.hpp
+++ b/src/client/clfunc_xsyr2.hpp
@@ -198,15 +198,15 @@ void xSyr2<T>::initialize_gpu_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.X, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.X, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuX, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer.Y, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.Y, CL_TRUE, 0,
buffer.N*sizeof(T),
buffer.cpuY, 0, NULL, NULL);
}
@@ -215,7 +215,7 @@ template <typename T>
void xSyr2<T>::reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer.A, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer.A, CL_TRUE,
buffer.offa * sizeof(T),
buffer.N * buffer.lda*sizeof(T),
buffer.cpuA, 0, NULL, NULL);;
@@ -226,7 +226,7 @@ void xSyr2<cl_float>::call_func()
{
timer.Start(timer_id);
clblasSsyr2(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -236,7 +236,7 @@ void xSyr2<cl_double>::call_func()
{
timer.Start(timer_id);
clblasSsyr2(buffer.order, buffer.uplo, buffer.N, buffer.alpha, buffer.X, buffer.offx,
- buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, 1, &queue_, 0, NULL,&event_);
+ buffer.incx, buffer.Y, buffer.offy, buffer.incy, buffer.A, buffer.offa, buffer.lda, numQueues, queues_, 0, NULL,&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xsyr2k.hpp b/src/client/clfunc_xsyr2k.hpp
index ae60f9e..b73f77a 100644
--- a/src/client/clfunc_xsyr2k.hpp
+++ b/src/client/clfunc_xsyr2k.hpp
@@ -324,19 +324,19 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -347,7 +347,7 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE, 0,
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
buffer_.c_, 0, NULL, NULL);
@@ -355,7 +355,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -603,7 +603,7 @@ call_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -634,8 +634,8 @@ roundtrip_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(float),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(float),
@@ -655,7 +655,7 @@ call_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -686,8 +686,8 @@ roundtrip_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(double),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(double),
@@ -707,7 +707,7 @@ call_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -738,8 +738,8 @@ roundtrip_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(cl_float2),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(cl_float2),
@@ -774,7 +774,7 @@ call_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -804,8 +804,8 @@ roundtrip_func()
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(cl_double2),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(cl_double2),
diff --git a/src/client/clfunc_xsyrk.hpp b/src/client/clfunc_xsyrk.hpp
index e9b6a7a..7b7d1d8 100644
--- a/src/client/clfunc_xsyrk.hpp
+++ b/src/client/clfunc_xsyrk.hpp
@@ -249,13 +249,13 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -266,7 +266,7 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_ * sizeof(T),
buffer_.ldc_ * buffer_.c_num_vectors_ *
sizeof(T),
@@ -275,7 +275,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_*sizeof(T), buffer_.ldc_*buffer_.c_num_vectors_*sizeof(T),
buffer_.c_, 0, NULL, NULL);
}
@@ -458,7 +458,7 @@ call_func()
clblasSsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, 4, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -484,8 +484,8 @@ xSyrk<float>::roundtrip_func()
clblasSsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_*sizeof(float), buffer_.ldc_*buffer_.c_num_vectors_*sizeof(float),
buffer_.c_, 0, NULL, &event_);
@@ -503,7 +503,7 @@ call_func()
clblasDsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -529,8 +529,8 @@ xSyrk<double>::roundtrip_func()
clblasDsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_*sizeof(double), buffer_.ldc_*buffer_.c_num_vectors_*sizeof(double),
buffer_.c_, 0, NULL, &event_);
@@ -548,7 +548,7 @@ call_func()
clblasCsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -574,8 +574,8 @@ xSyrk<cl_float2>::roundtrip_func()
clblasCsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_*sizeof(cl_float2), buffer_.ldc_*buffer_.c_num_vectors_*sizeof(cl_float2),
buffer_.c_, 0, NULL, &event_);
@@ -606,7 +606,7 @@ call_func()
clblasZsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, &event_);
+ buffer_.ldc_, numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -632,8 +632,8 @@ xSyrk<cl_double2>::roundtrip_func()
clblasZsyrk(order_, buffer_.uplo_, buffer_.trans_a_, buffer_.n_,
buffer_.k_, buffer_.alpha_, buffer_.buf_a_, buffer_.offA_,
buffer_.lda_, buffer_.beta_, buffer_.buf_c_, buffer_.offC_,
- buffer_.ldc_, 1, &queue_, 0, NULL, NULL);
- err = clEnqueueReadBuffer(queue_, buffer_.buf_c_, CL_TRUE,
+ buffer_.ldc_, numQueues, queues_, 0, NULL, NULL);
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_c_, CL_TRUE,
buffer_.offC_*sizeof(cl_double2), buffer_.ldc_*buffer_.c_num_vectors_*sizeof(cl_double2),
buffer_.c_, 0, NULL, &event_);
diff --git a/src/client/clfunc_xtrmm.hpp b/src/client/clfunc_xtrmm.hpp
index 2e05300..92d883c 100644
--- a/src/client/clfunc_xtrmm.hpp
+++ b/src/client/clfunc_xtrmm.hpp
@@ -272,13 +272,13 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ *buffer_.b_num_vectors_ *
sizeof(T),
@@ -288,7 +288,7 @@ public:
void reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
@@ -297,7 +297,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, NULL);
@@ -483,7 +483,7 @@ call_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -507,13 +507,13 @@ roundtrip_func()
buffer_.offB_) * sizeof(cl_float),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(cl_float),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(cl_float),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_float),
buffer_.ldb_ *buffer_.b_num_vectors_ *
sizeof(cl_float),
@@ -524,9 +524,9 @@ roundtrip_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, NULL);
+ numQueues, queues_, 0, NULL, NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_float), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(cl_float),
buffer_.b_, 0, NULL, &event_);
@@ -547,7 +547,7 @@ call_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offB_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -571,13 +571,13 @@ roundtrip_func()
buffer_.offB_) * sizeof(cl_double),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(cl_double),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(cl_double),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_double),
buffer_.ldb_ *buffer_.b_num_vectors_ *
sizeof(cl_double),
@@ -588,9 +588,9 @@ roundtrip_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, NULL);
+ numQueues, queues_, 0, NULL, NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_double), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(cl_double),
buffer_.b_, 0, NULL, &event_);
@@ -611,7 +611,7 @@ call_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -635,13 +635,13 @@ roundtrip_func()
buffer_.offB_) * sizeof(cl_float2),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(cl_float2),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(cl_float2),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_float2),
buffer_.ldb_ *buffer_.b_num_vectors_ *
sizeof(cl_float2),
@@ -652,9 +652,9 @@ roundtrip_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, NULL);
+ numQueues, queues_, 0, NULL, NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_float2), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(cl_float2),
buffer_.b_, 0, NULL, &event_);
@@ -675,7 +675,7 @@ call_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -699,13 +699,13 @@ roundtrip_func()
buffer_.offB_) * sizeof(cl_double2),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(cl_double2),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(cl_double2),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_double2),
buffer_.ldb_ *buffer_.b_num_vectors_ *
sizeof(cl_double2),
@@ -716,9 +716,9 @@ roundtrip_func()
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, NULL);
+ numQueues, queues_, 0, NULL, NULL);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(cl_double2), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(cl_double2),
buffer_.b_, 0, NULL, &event_);
diff --git a/src/client/clfunc_xtrmv.hpp b/src/client/clfunc_xtrmv.hpp
index 80d5004..bb1569f 100644
--- a/src/client/clfunc_xtrmv.hpp
+++ b/src/client/clfunc_xtrmv.hpp
@@ -195,11 +195,11 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE, 0,
buffer_.lda_*buffer_.a_num_vectors_*sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_x_, CL_TRUE, 0,
buffer_.m_*sizeof(T),
buffer_.x_, 0, NULL, NULL);
}
@@ -207,7 +207,7 @@ public:
void reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_x_, CL_TRUE, 0,
buffer_.m_,
buffer_.x_, 0, NULL, NULL);
}
@@ -264,7 +264,7 @@ call_func()
clblasStrmv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.scratch_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -278,7 +278,7 @@ call_func()
clblasDtrmv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.scratch_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -292,7 +292,7 @@ call_func()
clblasCtrmv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.scratch_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -306,7 +306,7 @@ call_func()
clblasZtrmv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
buffer_.lda_, buffer_.buf_x_, 0, 1, buffer_.scratch_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
diff --git a/src/client/clfunc_xtrsm.hpp b/src/client/clfunc_xtrsm.hpp
index cf6b269..20705ea 100644
--- a/src/client/clfunc_xtrsm.hpp
+++ b/src/client/clfunc_xtrsm.hpp
@@ -282,13 +282,13 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
@@ -298,7 +298,7 @@ public:
void reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
@@ -307,7 +307,7 @@ public:
void read_gpu_buffer()
{
cl_int err;
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, NULL);
@@ -327,13 +327,13 @@ public:
buffer_.offB_) * sizeof(T),
NULL, &err);
//initialize gpu buffer
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE,
buffer_.offA_ * sizeof(T),
buffer_.lda_ * buffer_.a_num_vectors_ *
sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T),
buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
@@ -341,7 +341,7 @@ public:
//call func
xTrsm_Function(false);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, &event_);
@@ -364,11 +364,11 @@ public:
NULL, &err);
// Map the buffers to pointers at host device
T *map_a,*map_b;
- map_a = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_a = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.ldb_ * buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
- map_b = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_b = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.ldb_ * buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
@@ -376,17 +376,17 @@ public:
memcpy( map_a, buffer_.a_, ( buffer_.lda_*buffer_.a_num_vectors_ + buffer_.offA_) * sizeof( T ) );
memcpy( map_b, buffer_.b_, ( buffer_.ldb_*buffer_.b_num_vectors_ + buffer_.offB_) * sizeof( T ) );
// unmap the buffers
- clEnqueueUnmapMemObject(queue_, buffer_.buf_a_, map_a, 0, NULL, NULL);
- clEnqueueUnmapMemObject(queue_, buffer_.buf_b_, map_b, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_a_, map_a, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_b_, map_b, 0, NULL, NULL);
//call func
xTrsm_Function(false);
// map the B buffer again to read the output
- map_b = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_b_, CL_TRUE, CL_MAP_READ, 0,
+ map_b = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_b_, CL_TRUE, CL_MAP_READ, 0,
(buffer_.ldb_ * buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
memcpy( map_b, buffer_.b_, ( buffer_.ldb_*buffer_.b_num_vectors_ + buffer_.offB_) * sizeof( T ) );
- clEnqueueUnmapMemObject(queue_, buffer_.buf_b_, map_b, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_b_, map_b, 0, NULL, NULL);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
}
@@ -407,7 +407,7 @@ public:
//call func
xTrsm_Function(false);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, &event_);
@@ -431,7 +431,7 @@ public:
//call func
xTrsm_Function(false);
//read gpu buffer
- err = clEnqueueReadBuffer(queue_, buffer_.buf_b_, CL_TRUE,
+ err = clEnqueueReadBuffer(queues_[0], buffer_.buf_b_, CL_TRUE,
buffer_.offB_ * sizeof(T), buffer_.ldb_ * buffer_.b_num_vectors_ *
sizeof(T),
buffer_.b_, 0, NULL, &event_);
@@ -455,11 +455,11 @@ public:
NULL, &err);
// Map the buffers to pointers at host device
T *map_a,*map_b;
- map_a = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_a = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_a_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.ldb_ * buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
- map_b = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
+ map_b = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_b_, CL_TRUE, CL_MAP_WRITE, 0,
(buffer_.ldb_ * buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
@@ -467,17 +467,17 @@ public:
memcpy( map_a, buffer_.a_, ( buffer_.lda_*buffer_.a_num_vectors_ + buffer_.offA_) * sizeof( T ) );
memcpy( map_b, buffer_.b_, ( buffer_.ldb_*buffer_.b_num_vectors_ + buffer_.offB_) * sizeof( T ) );
// unmap the buffers
- clEnqueueUnmapMemObject(queue_, buffer_.buf_a_, map_a, 0, NULL, NULL);
- clEnqueueUnmapMemObject(queue_, buffer_.buf_b_, map_b, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_a_, map_a, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_b_, map_b, 0, NULL, NULL);
//call func
xTrsm_Function(false);
// map the B buffer again to read the output
- map_b = (T*)clEnqueueMapBuffer(queue_, buffer_.buf_b_, CL_TRUE, CL_MAP_READ, 0,
+ map_b = (T*)clEnqueueMapBuffer(queues_[0], buffer_.buf_b_, CL_TRUE, CL_MAP_READ, 0,
(buffer_.ldb_ * buffer_.b_num_vectors_ +
buffer_.offB_) * sizeof(T),
0, NULL, NULL, &err);
memcpy( map_b, buffer_.b_, ( buffer_.ldb_*buffer_.b_num_vectors_ + buffer_.offB_) * sizeof( T ) );
- clEnqueueUnmapMemObject(queue_, buffer_.buf_b_, map_b, 0, NULL, NULL);
+ clEnqueueUnmapMemObject(queues_[0], buffer_.buf_b_, map_b, 0, NULL, NULL);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
#else
@@ -659,7 +659,7 @@ xTrsm_Function(bool flush)
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
if(flush==true)
{
clWaitForEvents(1, &event_);
@@ -676,7 +676,7 @@ xTrsm_Function(bool flush)
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
if(flush==true)
{
clWaitForEvents(1, &event_);
@@ -693,7 +693,7 @@ xTrsm_Function(bool flush)
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
if(flush==true)
{
clWaitForEvents(1, &event_);
@@ -710,7 +710,7 @@ xTrsm_Function(bool flush)
buffer_.m_, buffer_.n_, buffer_.alpha_,
buffer_.buf_a_, buffer_.offA_, buffer_.lda_,
buffer_.buf_b_, buffer_.offB_, buffer_.ldb_,
- 1, &queue_, 0, NULL, &event_);
+ numQueues, queues_, 0, NULL, &event_);
if(flush==true)
{
clWaitForEvents(1, &event_);
diff --git a/src/client/clfunc_xtrsv.hpp b/src/client/clfunc_xtrsv.hpp
index 4eb0e5b..3048880 100644
--- a/src/client/clfunc_xtrsv.hpp
+++ b/src/client/clfunc_xtrsv.hpp
@@ -188,11 +188,11 @@ public:
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_a_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_a_, CL_TRUE, 0,
buffer_.lda_*buffer_.a_num_vectors_*sizeof(T),
buffer_.a_, 0, NULL, NULL);
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_x_, CL_TRUE, 0,
buffer_.m_*sizeof(T),
buffer_.x_, 0, NULL, NULL);
}
@@ -200,7 +200,7 @@ public:
void reset_gpu_write_buffer()
{
cl_int err;
- err = clEnqueueWriteBuffer(queue_, buffer_.buf_x_, CL_TRUE, 0,
+ err = clEnqueueWriteBuffer(queues_[0], buffer_.buf_x_, CL_TRUE, 0,
buffer_.m_,
buffer_.x_, 0, NULL, NULL);
}
@@ -256,7 +256,7 @@ call_func()
timer.Start(timer_id);
clblasStrsv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
- buffer_.lda_, buffer_.buf_x_, 0, 1, 1, &queue_, 0, NULL,
+ buffer_.lda_, buffer_.buf_x_, 0, 1, numQueues, queues_, 0, NULL,
&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -270,7 +270,7 @@ call_func()
timer.Start(timer_id);
clblasDtrsv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
- buffer_.lda_, buffer_.buf_x_, 0, 1, 1, &queue_, 0, NULL,
+ buffer_.lda_, buffer_.buf_x_, 0, 1, numQueues, queues_, 0, NULL,
&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -284,7 +284,7 @@ call_func()
timer.Start(timer_id);
clblasCtrsv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
- buffer_.lda_, buffer_.buf_x_, 0, 1, 1, &queue_, 0, NULL,
+ buffer_.lda_, buffer_.buf_x_, 0, 1, numQueues, queues_, 0, NULL,
&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
@@ -298,7 +298,7 @@ call_func()
timer.Start(timer_id);
clblasZtrsv(order_, buffer_.uplo_, buffer_.trans_a_,
buffer_.diag_, buffer_.m_, buffer_.buf_a_, 0,
- buffer_.lda_, buffer_.buf_x_, 0, 1, 1, &queue_, 0, NULL,
+ buffer_.lda_, buffer_.buf_x_, 0, 1, numQueues, queues_, 0, NULL,
&event_);
clWaitForEvents(1, &event_);
timer.Stop(timer_id);
diff --git a/src/client/client.cpp b/src/client/client.cpp
index 271a28d..59ec8e5 100644
--- a/src/client/client.cpp
+++ b/src/client/client.cpp
@@ -74,6 +74,7 @@ int main(int argc, char *argv[])
int side_option;
int uplo_option;
int diag_option;
+ unsigned int numQueuesToUse;
po::options_description desc( "clBLAS client command line options" );
desc.add_options()
@@ -103,6 +104,7 @@ int main(int argc, char *argv[])
( "diag", po::value<int>( &diag_option )->default_value(0), "0 = unit diagonal, 1 = non unit diagonal. only used with [list of function families]" ) // xtrsm xtrmm
( "profile,p", po::value<cl_uint>( &profileCount )->default_value(20), "Time and report the kernel speed (default: 20)" )
( "apiCallCount", po::value<cl_uint>(&apiCallCount)->default_value(10), "Time and report the kernel speed on counds of API calls (default: 10)")
+ ( "numQueues", po::value<unsigned int>(&numQueuesToUse)->default_value(1), "Number of cl_command_queues to use( default: 1)")
( "roundtrip", po::value<std::string>( &roundtrip )->default_value("noroundtrip"),"including the time of OpenCL memory allocation and transportation; options:roundtrip, noroundtrip(default)")
( "memalloc", po::value<std::string>( &memalloc )->default_value("default"),"setting the memory allocation flags for OpenCL; would not take effect if roundtrip time is not measured; options:default(default),alloc_host_ptr,use_host_ptr,copy_host_ptr,use_persistent_mem_amd,rect_mem")
;
@@ -195,13 +197,13 @@ int main(int argc, char *argv[])
if (function == "gemm")
{
if (precision == "s")
- my_function = new xGemm<cl_float>(timer, deviceType);
+ my_function = new xGemm<cl_float>(timer, deviceType, numQueuesToUse);
else if (precision == "d")
- my_function = new xGemm<cl_double>(timer, deviceType);
+ my_function = new xGemm<cl_double>(timer, deviceType, numQueuesToUse);
else if (precision == "c")
- my_function = new xGemm<cl_float2>(timer, deviceType);
+ my_function = new xGemm<cl_float2>(timer, deviceType, numQueuesToUse);
else if (precision == "z")
- my_function = new xGemm<cl_double2>(timer, deviceType);
+ my_function = new xGemm<cl_double2>(timer, deviceType, numQueuesToUse);
else
{
std::cerr << "Unknown gemm function" << std::endl;
diff --git a/src/library/CMakeLists.txt b/src/library/CMakeLists.txt
index fa195bc..325644e 100644
--- a/src/library/CMakeLists.txt
+++ b/src/library/CMakeLists.txt
@@ -14,6 +14,285 @@
# limitations under the License.
# ########################################################################
+
+
+
+################################################################################
+# AutoGemm Begin
+################################################################################
+
+# AutoGemm scripts and out files
+set(AUTOGEMM_SCRIPTS
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/AutoGemm.py
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/AutoGemmParameters.py
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/Common.py
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/Includes.py
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/KernelOpenCL.py
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/KernelParameters.py
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/KernelSelection.py
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/KernelsToPreCompile.py
+)
+set(AUTOGEMM_HEADERS
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmClKernels.h
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelBuildOptionsBinary.h
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelBinaries.h
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelSelection.h
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelSelectionSpecific.h
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelBuildOptionsSource.h
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelSources.h
+)
+
+set(AUTOGEMM_SRC
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmClKernels.cpp
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelBuildOptionsBinary.cpp
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelBinaries.cpp
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelSelection.cpp
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelSelectionSpecific.cpp
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelBuildOptionsSource.cpp
+ ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelSources.cpp
+)
+
+#set(USERGEMM_SRC
+# ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.cpp
+#)
+
+set(USERGEMM_HEADERS
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.h
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/UserGemmKernelSources/UserGemmClKernels.h
+)
+
+set(AUTOGEMM_TEST_SRC
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/AutoGemmTools/TestAutoGemm.cpp
+)
+set(AUTOGEMM_PROFILER_SRC
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/AutoGemmTools/ProfileAutoGemm.cpp
+)
+set(AUTOGEMM_PRECOMPILE_SRC
+ ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/AutoGemmTools/AutoGemmPreCompileKernels.cpp
+)
+set( AUTOGEMM_PRECOMPILED_KERNELS
+ ${CMAKE_BINARY_DIR}/include/AutoGemmKernelBinaries/AutoGemmKernelBinariesPreCompiled.h
+)
+
+# AutoGemm options for pre-compiling kernels
+option( PRECOMPILE_GEMM_PRECISION_SGEMM "AutoGemm: pre-compile sgemm kernels" OFF)
+option( PRECOMPILE_GEMM_PRECISION_DGEMM "AutoGemm: pre-compile dgemm kernels" OFF)
+option( PRECOMPILE_GEMM_PRECISION_CGEMM "AutoGemm: pre-compile cgemm kernels" OFF)
+option( PRECOMPILE_GEMM_PRECISION_ZGEMM "AutoGemm: pre-compile zgemm kernels" OFF)
+
+option( PRECOMPILE_GEMM_TRANS_NN "AutoGemm: pre-compile NN transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_NT "AutoGemm: pre-compile NT transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_NC "AutoGemm: pre-compile NC transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_TN "AutoGemm: pre-compile TN transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_TT "AutoGemm: pre-compile TT transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_TC "AutoGemm: pre-compile TC transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_CN "AutoGemm: pre-compile CN transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_CT "AutoGemm: pre-compile CT transpose cases" OFF)
+option( PRECOMPILE_GEMM_TRANS_CC "AutoGemm: pre-compile CC transpose cases" OFF)
+
+# opencl compiler version
+#set( PRECOMPILE_GEMM_OPENCL_VERSION "2.0" CACHE STRING "OpenCL compiler version supported by device driver." )
+#set_property( CACHE PRECOMPILE_GEMM_OPENCL_VERSION PROPERTY STRINGS 2.0 1.2 1.1 )
+#message( STATUS "AutoGemm PreCompiler will use OpenCL ${PRECOMPILE_GEMM_OPENCL_VERSION} compiler." )
+
+# PreCompile precision selected?
+set( PRECOMPILE_GEMM_PRECISION_SELECTED OFF)
+if ( PRECOMPILE_GEMM_PRECISION_SGEMM
+ OR PRECOMPILE_GEMM_PRECISION_DGEMM
+ OR PRECOMPILE_GEMM_PRECISION_CGEMM
+ OR PRECOMPILE_GEMM_PRECISION_ZGEMM )
+ set( PRECOMPILE_GEMM_PRECISION_SELECTED ON)
+endif()
+
+# PreCompile transpose selected?
+set( PRECOMPILE_GEMM_TRANS_SELECTED OFF)
+if ( PRECOMPILE_GEMM_TRANS_NN
+ OR PRECOMPILE_GEMM_TRANS_NT
+ OR PRECOMPILE_GEMM_TRANS_NC
+ OR PRECOMPILE_GEMM_TRANS_TN
+ OR PRECOMPILE_GEMM_TRANS_TT
+ OR PRECOMPILE_GEMM_TRANS_TC
+ OR PRECOMPILE_GEMM_TRANS_CN
+ OR PRECOMPILE_GEMM_TRANS_CT
+ OR PRECOMPILE_GEMM_TRANS_CC )
+ set( PRECOMPILE_GEMM_TRANS_SELECTED ON)
+endif()
+
+# PreCompile is valid and active?
+set( PRECOMPILE_GEMM_ACTIVE OFF)
+if ( PRECOMPILE_GEMM_PRECISION_SELECTED
+ AND PRECOMPILE_GEMM_TRANS_SELECTED)
+ # valid selection
+ set( PRECOMPILE_GEMM_ACTIVE ON)
+ MESSAGE( STATUS "AutoGemm-PreCompile: selected kernels will be pre-compiled." )
+elseif(NOT PRECOMPILE_GEMM_PRECISION_SELECTED
+ AND NOT PRECOMPILE_GEMM_TRANS_SELECTED)
+ MESSAGE( STATUS "AutoGemm-PreCompile: no kernels to be pre-compiled." )
+else()
+ MESSAGE( SEND_ERROR "AutoGemm-PreCompile: To pre-compile gemm kernels, select at lease one option from each of PRECOMPILE_GEMM_PRECISION_* and PRECOMPILE_GEMM_TRANS_*; otherwise, unselect all PRECOMPILE_GEMM_* options to not pre-compile any gemm kernels." )
+endif()
+
+# build commandline argument for AutoGemm
+set( AGPC_ARGS --output ${CMAKE_BINARY_DIR}/include )
+
+if ( PRECOMPILE_GEMM_ACTIVE )
+ # precisions
+ set(AGPC_ARGS ${AGPC_ARGS} --precisions )
+ if (PRECOMPILE_GEMM_PRECISION_SGEMM)
+ set(AGPC_ARGS ${AGPC_ARGS} s )
+ endif()
+ if (PRECOMPILE_GEMM_PRECISION_DGEMM)
+ set(AGPC_ARGS ${AGPC_ARGS} d )
+ endif()
+ if (PRECOMPILE_GEMM_PRECISION_CGEMM)
+ set(AGPC_ARGS ${AGPC_ARGS} c )
+ endif()
+ if (PRECOMPILE_GEMM_PRECISION_ZGEMM)
+ set(AGPC_ARGS ${AGPC_ARGS} z )
+ endif()
+
+ # orders
+ set(AGPC_ARGS ${AGPC_ARGS} --orders clblasColumnMajor )
+
+ # transposes
+ set(AGPC_ARGS ${AGPC_ARGS} --transposes )
+ if (PRECOMPILE_GEMM_TRANS_NN)
+ set(AGPC_ARGS ${AGPC_ARGS} NN )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_NT)
+ set(AGPC_ARGS ${AGPC_ARGS} NT )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_NC)
+ set(AGPC_ARGS ${AGPC_ARGS} NC )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_TN)
+ set(AGPC_ARGS ${AGPC_ARGS} TN )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_TT)
+ set(AGPC_ARGS ${AGPC_ARGS} TT )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_TC)
+ set(AGPC_ARGS ${AGPC_ARGS} TC )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_CN)
+ set(AGPC_ARGS ${AGPC_ARGS} CN )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_CT)
+ set(AGPC_ARGS ${AGPC_ARGS} CT )
+ endif()
+ if (PRECOMPILE_GEMM_TRANS_CC)
+ set(AGPC_ARGS ${AGPC_ARGS} CC )
+ endif()
+
+ # betas
+ set(AGPC_ARGS ${AGPC_ARGS} --betas 0 1 )
+
+
+
+################################################################################
+# add target for generating pre-compile WhichKernels header
+################################################################################
+set( AUTOGEMM_PRECOMPILE_HEADER_SRC ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/KernelsToPreCompile.py )
+set( AUTOGEMM_PRECOMPILE_HEADER_OUT ${CMAKE_BINARY_DIR}/include/AutoGemmIncludes/AutoGemmKernelsToPreCompile.h )
+add_custom_command(
+ OUTPUT ${AUTOGEMM_PRECOMPILE_HEADER_OUT}
+ COMMAND python ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/KernelsToPreCompile.py ${AGPC_ARGS}
+ DEPENDS ${AUTOGEMM_PRECOMPILE_HEADER_SRC}
+)
+
+################################################################################
+# add target for compiling pre-compile executable
+################################################################################
+add_executable(AutoGemm_PreCompile_Bin
+ ${AUTOGEMM_PRECOMPILE_SRC}
+ ${AUTOGEMM_PRECOMPILE_HEADER_OUT}
+ ${AUTOGEMM_HEADERS}
+ ${AUTOGEMM_SRC}
+ ${AUTOGEMM_SCRIPTS}
+ )
+target_link_libraries(AutoGemm_PreCompile_Bin ${OPENCL_LIBRARIES})
+set_property( TARGET AutoGemm_PreCompile_Bin PROPERTY FOLDER "AutoGemm")
+#set_target_properties(
+ #AutoGemm_PreCompile_Bin
+ #PROPERTIES
+ #EXCLUDE_FROM_ALL TRUE
+ #EXCLUDE_FROM_DEFAULT_BUILD TRUE
+#)
+
+
+################################################################################
+# add target for running pre-compile executable
+################################################################################
+add_custom_command(
+ OUTPUT ${AUTOGEMM_PRECOMPILED_KERNELS}
+ COMMAND AutoGemm_PreCompile_Bin ${CMAKE_BINARY_DIR}
+ DEPENDS AutoGemm_PreCompile_Bin
+)
+
+endif()#endif precompile active
+
+
+################################################################################
+# add target for main AutoGemm headers / source
+################################################################################
+add_custom_command(
+ OUTPUT ${AUTOGEMM_HEADERS} ${AUTOGEMM_SRC}
+ COMMAND python ${CMAKE_SOURCE_DIR}/library/blas/AutoGemm/AutoGemm.py --output-path ${CMAKE_BINARY_DIR}/include --opencl-compiler-version ${OPENCL_VERSION}
+ DEPENDS ${AUTOGEMM_SCRIPTS}
+)
+
+
+include_directories(
+ ${OPENCL_INCLUDE_DIRS}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_BINARY_DIR}/include
+ .
+)
+
+################################################################################
+# AutoGemm Tools
+################################################################################
+add_executable(AutoGemm_Tools_Test
+ ${AUTOGEMM_TEST_SRC}
+ ${AUTOGEMM_SRC}
+ ${AUTOGEMM_HEADERS}
+ ${AUTOGEMM_SCRIPTS}
+ )
+target_link_libraries(AutoGemm_Tools_Test ${OPENCL_LIBRARIES})
+set_property( TARGET AutoGemm_Tools_Test PROPERTY FOLDER "AutoGemm")
+set_target_properties(
+ AutoGemm_Tools_Test
+ PROPERTIES
+ EXCLUDE_FROM_ALL TRUE
+ EXCLUDE_FROM_DEFAULT_BUILD TRUE
+)
+
+
+add_executable(AutoGemm_Tools_Profile
+ ${AUTOGEMM_PROFILER_SRC}
+ ${AUTOGEMM_SRC}
+ ${AUTOGEMM_HEADERS}
+ ${AUTOGEMM_SCRIPTS}
+ )
+target_link_libraries(AutoGemm_Tools_Profile ${OPENCL_LIBRARIES})
+set_property( TARGET AutoGemm_Tools_Profile PROPERTY FOLDER "AutoGemm")
+set_target_properties(
+ AutoGemm_Tools_Profile
+ PROPERTIES
+ EXCLUDE_FROM_ALL TRUE
+ EXCLUDE_FROM_DEFAULT_BUILD TRUE
+)
+
+source_group(AutoGemm\\scripts FILES ${AUTOGEMM_SCRIPTS} )
+source_group(AutoGemm\\include FILES ${AUTOGEMM_HEADERS} )
+source_group(AutoGemm\\src FILES ${AUTOGEMM_SRC} ${AUTOGEMM_PRECOMPILED_KERNELS} )
+
+################################################################################
+# AutoGemm End
+################################################################################
+
+
+
set(SRC_BLAS
blas/init.c
blas/impl.c
@@ -82,6 +361,7 @@ set(SRC_BLAS
blas/functor/gcn_sgemmSmallMatrices.cc
blas/functor/hawaii_sgemmBranchKernel.cc
blas/functor/hawaii_sgemmBig1024Kernel.cc
+ blas/specialCases/GemmSpecialCases.cpp
)
set(SRC_BLAS_HEADERS
@@ -92,6 +372,7 @@ set(SRC_BLAS_HEADERS
blas/include/clblas-internal.h
blas/include/solution_seq.h
blas/include/events.h
+ blas/include/xgemm.h
blas/functor/include/functor.h
blas/functor/include/functor_xgemm.h
blas/functor/include/functor_xscal.h
@@ -116,6 +397,8 @@ set(SRC_BLAS_HEADERS
blas/functor/include/gcn_sgemmSmallMatrices.h
blas/functor/include/hawaii_sgemmBranchKernel.h
blas/functor/include/hawaii_sgemmBig1024Kernel.h
+ blas/AutoGemm/UserGemmKernelSources/UserGemmClKernels.h
+ blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.h
)
set(SRC_BLAS_GENERIC
@@ -346,8 +629,7 @@ source_group(common\\gens FILES ${SRC_COMMON_GENS})
source_group(blas FILES ${SRC_BLAS})
source_group(blas\\include FILES ${SRC_BLAS_HEADERS})
source_group(blas\\generic FILES ${SRC_BLAS_GENERIC})
-source_group(blas\\gens FILES ${SRC_BLAS_GENS}
- ${SRC_BLAS_GENS_HEADERS})
+source_group(blas\\gens FILES ${SRC_BLAS_GENS} ${SRC_BLAS_GENS_HEADERS})
include_directories(${OPENCL_INCLUDE_DIRS}
${clBLAS_SOURCE_DIR}
@@ -356,6 +638,9 @@ include_directories(${OPENCL_INCLUDE_DIRS}
${clBLAS_SOURCE_DIR}/library/blas/functor/include
${clBLAS_SOURCE_DIR}/library/tools/tune
${clBLAS_BINARY_DIR}/include
+ ${clBLAS_SOURCE_DIR}/library/blas/AutoGemm
+ ${clBLAS_SOURCE_DIR}/library/blas/AutoGemm/UserGemmKernelSources
+ ${clBLAS_SOURCE_DIR}/library/blas/specialCases/include
)
option( BLAS_DUMP_CLBLAS_KERNELS "Force the library to dump OpenCL kernels to disk" OFF )
@@ -385,7 +670,7 @@ ExternalProject_Add( tplgen
)
# if offline compilation is not chosen, bingen should not be built
-if(OCL_OFFLINE_BUILD_TAHITI_KERNEL OR OCL_OFFLINE_BUILD_HAWAII_KERNEL OR OCL_OFFLINE_BUILD_BONAIRE_KERNEL)
+if(OPENCL_OFFLINE_BUILD_TAHITI_KERNEL OR OPENCL_OFFLINE_BUILD_HAWAII_KERNEL OR OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL)
ExternalProject_Add( bingen
URL "${CMAKE_SOURCE_DIR}/library/tools/bingen"
CMAKE_ARGS -DOPENCL_LIBRARIES=${OPENCL_LIBRARIES} -DOPENCL_INCLUDE_DIRS=${OPENCL_INCLUDE_DIRS}
@@ -393,41 +678,42 @@ if(OCL_OFFLINE_BUILD_TAHITI_KERNEL OR OCL_OFFLINE_BUILD_HAWAII_KERNEL OR OCL_OFF
)
endif()
-message(STATUS "OCL_VERSION = ${OCL_VERSION}")
-if( OCL_VERSION STREQUAL "2.0")
- if(EXISTS ${CMAKE_SOURCE_DIR}/flags.txt)
- MESSAGE(STATUS "flags.txt found. will load AMD_OCL_BUILD_OPTIONS_APPEND from it.")
- set (LOAD_CL_FLAGS TRUE)
- file (STRINGS "${CMAKE_SOURCE_DIR}/flags.txt" OCL_FLAGS)
- MESSAGE(STATUS "OCLFLAGS: ${OCL_FLAGS}")
- string(REPLACE "OCL " "OCL;" OCL_FLAGS_REPLACED ${OCL_FLAGS})
- list(GET OCL_FLAGS_REPLACED 1 OCL_FLAGS_REPLACED_1)#flags for TAHITI
- list(GET OCL_FLAGS_REPLACED 3 OCL_FLAGS_REPLACED_3)#flags for HAWAII 1
- list(GET OCL_FLAGS_REPLACED 5 OCL_FLAGS_REPLACED_5)#flags for HAWAII 2
- list(GET OCL_FLAGS_REPLACED 7 OCL_FLAGS_REPLACED_7)#flags for BONAIRE
- #MESSAGE("${OCL_FLAGS_REPLACED_7}")
- elseif(EXISTS ${CMAKE_SOURCE_DIR}/flags_public.txt)
- MESSAGE(STATUS "flags_public.txt found. will load AMD_OCL_BUILD_OPTIONS_APPEND from it.")
- set (LOAD_CL_FLAGS TRUE)
- file (STRINGS "${CMAKE_SOURCE_DIR}/flags_public.txt" OCL_FLAGS)
- MESSAGE(STATUS "OCLFLAGS: ${OCL_FLAGS}")
- string(REPLACE "OCL " "OCL;" OCL_FLAGS_REPLACED ${OCL_FLAGS})
- list(GET OCL_FLAGS_REPLACED 1 OCL_FLAGS_REPLACED_1)#flags for TAHITI
- list(GET OCL_FLAGS_REPLACED 3 OCL_FLAGS_REPLACED_3)#flags for HAWAII 1
- list(GET OCL_FLAGS_REPLACED 5 OCL_FLAGS_REPLACED_5)#flags for HAWAII 2
- list(GET OCL_FLAGS_REPLACED 7 OCL_FLAGS_REPLACED_7)#flags for BONAIRE
- else()
- MESSAGE(STATUS "flags.txt not found. will use the default flags.")
- set (LOAD_CL_FLAGS FALSE)
- endif()
-else()
- MESSAGE(STATUS "loading of compiler flags requires OpenCL 2.0. will use default flags.")
- set (LOAD_CL_FLAGS FALSE)
-endif()
+message(STATUS "OPENCL_VERSION = ${OPENCL_VERSION}")
+#if( OPENCL_VERSION STREQUAL "2.0")
+# if(EXISTS ${CMAKE_SOURCE_DIR}/flags.txt)
+# MESSAGE(STATUS "flags.txt found. will load AMD_OPENCL_BUILD_OPTIONS_APPEND from it.")
+# set (LOAD_CL_FLAGS TRUE)
+# file (STRINGS "${CMAKE_SOURCE_DIR}/flags.txt" OPENCL_FLAGS)
+# MESSAGE(STATUS "OCLFLAGS: ${OPENCL_FLAGS}")
+# string(REPLACE "OCL " "OCL;" OPENCL_FLAGS_REPLACED ${OPENCL_FLAGS})
+# list(GET OPENCL_FLAGS_REPLACED 1 OPENCL_FLAGS_REPLACED_1)#flags for TAHITI
+# list(GET OPENCL_FLAGS_REPLACED 3 OPENCL_FLAGS_REPLACED_3)#flags for HAWAII 1
+# list(GET OPENCL_FLAGS_REPLACED 5 OPENCL_FLAGS_REPLACED_5)#flags for HAWAII 2
+# list(GET OPENCL_FLAGS_REPLACED 7 OPENCL_FLAGS_REPLACED_7)#flags for BONAIRE
+# #MESSAGE("${OPENCL_FLAGS_REPLACED_7}")
+# elseif(EXISTS ${CMAKE_SOURCE_DIR}/flags_public.txt)
+# MESSAGE(STATUS "flags_public.txt found. will load AMD_OPENCL_BUILD_OPTIONS_APPEND from it.")
+# set (LOAD_CL_FLAGS TRUE)
+# file (STRINGS "${CMAKE_SOURCE_DIR}/flags_public.txt" OPENCL_FLAGS)
+# MESSAGE(STATUS "OCLFLAGS: ${OPENCL_FLAGS}")
+# string(REPLACE "OCL " "OCL;" OPENCL_FLAGS_REPLACED ${OPENCL_FLAGS})
+# list(GET OPENCL_FLAGS_REPLACED 1 OPENCL_FLAGS_REPLACED_1)#flags for TAHITI
+# list(GET OPENCL_FLAGS_REPLACED 3 OPENCL_FLAGS_REPLACED_3)#flags for HAWAII 1
+# list(GET OPENCL_FLAGS_REPLACED 5 OPENCL_FLAGS_REPLACED_5)#flags for HAWAII 2
+# list(GET OPENCL_FLAGS_REPLACED 7 OPENCL_FLAGS_REPLACED_7)#flags for BONAIRE
+# else()
+# MESSAGE(STATUS "flags.txt not found. will use the default flags.")
+# set (LOAD_CL_FLAGS FALSE)
+# endif()
+#else()
+# MESSAGE(STATUS "loading of compiler flags requires OpenCL 2.0. will use default flags.")
+# set (LOAD_CL_FLAGS FALSE)
+ #endif()
+set (LOAD_CL_FLAGS FALSE)
#set( bingenBinaryDir "${CMAKE_BINARY_DIR}/library/tools/bingen/staging" )
# if offline compilation is not chosen, bingen should not be built
-if(OCL_OFFLINE_BUILD_TAHITI_KERNEL OR OCL_OFFLINE_BUILD_HAWAII_KERNEL OR OCL_OFFLINE_BUILD_BONAIRE_KERNEL)
+if(OPENCL_OFFLINE_BUILD_TAHITI_KERNEL OR OPENCL_OFFLINE_BUILD_HAWAII_KERNEL OR OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL)
ExternalProject_Get_Property( bingen binary_dir )
set( bingenBinaryDir "" )
@@ -443,9 +729,9 @@ add_custom_target( GEN_CLBIN )
add_custom_command(TARGET GEN_CLBIN
PRE_BUILD
COMMAND ${CMAKE_COMMAND} -DbingenBinaryDir=${bingenBinaryDir} -DCLTEMPLATE_PATH="${CMAKE_SOURCE_DIR}/library/blas/gens/clTemplates"
- -DLOAD_CL_FLAGS=${LOAD_CL_FLAGS} -DTAHITI_FLAG=${OCL_FLAGS_REPLACED_1} -DHAWAII1_FLAG=${OCL_FLAGS_REPLACED_3} -DHAWAII2_FLAG=${OCL_FLAGS_REPLACED_5} -DBONAIRE_FLAG=${OCL_FLAGS_REPLACED_7}
- -DENV_PATH=${ENV_PATH} -DOCL_OFFLINE_BUILD_HAWAII_KERNEL=${OCL_OFFLINE_BUILD_HAWAII_KERNEL} -DOCL_OFFLINE_BUILD_BONAIRE_KERNEL=${OCL_OFFLINE_BUILD_BONAIRE_KERNEL}
- -DOCL_OFFLINE_BUILD_TAHITI_KERNEL=${OCL_OFFLINE_BUILD_TAHITI_KERNEL}
+ -DLOAD_CL_FLAGS=${LOAD_CL_FLAGS} -DTAHITI_FLAG=${OPENCL_FLAGS_REPLACED_1} -DHAWAII1_FLAG=${OPENCL_FLAGS_REPLACED_3} -DHAWAII2_FLAG=${OPENCL_FLAGS_REPLACED_5} -DBONAIRE_FLAG=${OPENCL_FLAGS_REPLACED_7}
+ -DENV_PATH=${ENV_PATH} -DOPENCL_OFFLINE_BUILD_HAWAII_KERNEL=${OPENCL_OFFLINE_BUILD_HAWAII_KERNEL} -DOPENCL_OFFLINE_BUILD_BONAIRE_KERNEL=${OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL}
+ -DOPENCL_OFFLINE_BUILD_TAHITI_KERNEL=${OPENCL_OFFLINE_BUILD_TAHITI_KERNEL}
-P "${CMAKE_SOURCE_DIR}/library/bingen.cmake"
)
add_dependencies( GEN_CLBIN bingen )
@@ -454,13 +740,13 @@ add_custom_target( GEN_CLBIN )
add_custom_command(TARGET GEN_CLBIN
PRE_BUILD
COMMAND ${CMAKE_COMMAND} -DbingenBinaryDir=${bingenBinaryDir} -DCLTEMPLATE_PATH="${CMAKE_SOURCE_DIR}/library/blas/gens/clTemplates"
- -DOCL_OFFLINE_BUILD_HAWAII_KERNEL=${OCL_OFFLINE_BUILD_HAWAII_KERNEL} -DOCL_OFFLINE_BUILD_BONAIRE_KERNEL=${OCL_OFFLINE_BUILD_BONAIRE_KERNEL}
- -DOCL_OFFLINE_BUILD_TAHITI_KERNEL=${OCL_OFFLINE_BUILD_TAHITI_KERNEL}
+ -DOPENCL_OFFLINE_BUILD_HAWAII_KERNEL=${OPENCL_OFFLINE_BUILD_HAWAII_KERNEL} -DOPENCL_OFFLINE_BUILD_BONAIRE_KERNEL=${OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL}
+ -DOPENCL_OFFLINE_BUILD_TAHITI_KERNEL=${OPENCL_OFFLINE_BUILD_TAHITI_KERNEL}
-P "${CMAKE_SOURCE_DIR}/library/bingen.cmake"
)
add_dependencies( GEN_CLBIN bingen )
endif()
-endif()#if(OCL_OFFLINE_BUILD_TAHITI_KERNEL OR OCL_OFFLINE_BUILD_HAWAII_KERNEL OR OCL_OFFLINE_BUILD_BONAIRE_KERNEL)
+endif()#if(OPENCL_OFFLINE_BUILD_TAHITI_KERNEL OR OPENCL_OFFLINE_BUILD_HAWAII_KERNEL OR OPENCL_OFFLINE_BUILD_BONAIRE_KERNEL)
ExternalProject_Get_Property( tplgen binary_dir )
@@ -478,7 +764,7 @@ add_custom_target( GENERATE_CLT
WORKING_DIRECTORY ${bingenBinaryDir}
)
-add_dependencies( tplgen GEN_CLBIN )
+#add_dependencies( tplgen GEN_CLBIN )
add_dependencies( GENERATE_CLT tplgen )
if( CMAKE_COMPILER_IS_GNUCC )
@@ -489,8 +775,36 @@ if( CMAKE_COMPILER_IS_GNUCC )
DESTINATION lib${SUFFIX_LIB}/pkgconfig )
endif( )
-add_library(clBLAS ${CLBLAS_SOURCES} ${GLOBAL_HEADERS} ${SRC_BLAS_HEADERS} ${SRC_BLAS_GENS_HEADERS})
+# clBLAS to depend on AutoGemm
+
+if ( ${PRECOMPILE_GEMM_ACTIVE} )
+ set( AUTOGEMM_PRECOMPILED_KERNELS_CONDITIONAL ${AUTOGEMM_PRECOMPILED_KERNELS} )
+ MESSAGE( STATUS "clBLAS will depend on ${AUTOGEMM_PRECOMPILED_KERNELS}" )
+else()
+ set( AUTOGEMM_PRECOMPILED_KERNELS_CONDITIONAL )
+ MESSAGE( STATUS "clBLAS will NOT depend on ${AUTOGEMM_PRECOMPILED_KERNELS}" )
+endif()
+
+
+add_library(clBLAS
+ ${CLBLAS_SOURCES}
+ ${GLOBAL_HEADERS}
+ ${SRC_BLAS_HEADERS}
+ ${SRC_BLAS_GENS_HEADERS}
+ ${AUTOGEMM_SRC}
+ ${AUTOGEMM_HEADERS}
+ ${AUTOGEMM_SCRIPTS}
+ ${AUTOGEMM_PRECOMPILED_KERNELS_CONDITIONAL}
+ #${USERGEMM_SRC}
+ #${USERGEMM_HEADERS}
+ )
add_dependencies(clBLAS GENERATE_CLT)
+
+# AutoGemm needs compiler flag to utilize pre-compiled kernels
+if ( ${PRECOMPILE_GEMM_ACTIVE} )
+ set_target_properties(clBLAS PROPERTIES COMPILE_FLAGS -DAUTOGEMM_USE_PRE_COMPILED_KERNELS)
+endif()
+
set_target_properties(clBLAS PROPERTIES VERSION ${clBLAS_VERSION})
set_target_properties(clBLAS PROPERTIES SOVERSION ${clBLAS_SOVERSION})
set_target_properties( clBLAS PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/staging" )
diff --git a/src/library/blas/AutoGemm/.gitignore b/src/library/blas/AutoGemm/.gitignore
new file mode 100644
index 0000000..21e56d4
--- /dev/null
+++ b/src/library/blas/AutoGemm/.gitignore
@@ -0,0 +1,4 @@
+*.cl
+*.swp
+*.txt
+*.pyc
diff --git a/src/library/blas/AutoGemm/AutoGemm.py b/src/library/blas/AutoGemm/AutoGemm.py
new file mode 100644
index 0000000..2bedeb4
--- /dev/null
+++ b/src/library/blas/AutoGemm/AutoGemm.py
@@ -0,0 +1,47 @@
+################################################################################
+# AutoGemm
+# - Automatically generate gemm kernels based on tile parameters
+# - This script generates the following to ease integration into clBLAS:
+# - generate all the kernel files
+# - kernel selection logic
+# - include files for kernel strings
+#
+# TODO Now
+# - offline compilation
+# TODO Future
+# - fuse together unroll=8 and unroll=1 in same kernel ?
+# functionally works fine, but lowers performance by ~10%
+################################################################################
+
+import os
+import sys
+import argparse
+import getopt
+
+import Common
+import Includes
+import KernelSelection
+import KernelOpenCL
+
+
+################################################################################
+# Main
+################################################################################
+if __name__ == "__main__":
+ # parse arguments
+ ap = argparse.ArgumentParser(description="AutoGemm")
+ ap.add_argument("--output-path", dest="output" )
+ ap.add_argument("--opencl-compiler-version", dest="clCompilerVersion", action="store", choices=["1.1", "1.2", "2.0" ])
+ args = ap.parse_args()
+ if args.output:
+ Common.setOutputPath(args.output)
+ else:
+ print "AutoGemm.py: Warning: No output path specified; default is working directory."
+
+ print "AutoGemm.py: using OpenCL " + args.clCompilerVersion + " compiler"
+ Common.setClCompilerVersion(args.clCompilerVersion)
+
+ KernelOpenCL.writeOpenCLKernels()
+ KernelSelection.writeKernelSelection()
+ Includes.writeIncludes()
+
diff --git a/src/library/blas/AutoGemm/AutoGemmParameters.py b/src/library/blas/AutoGemm/AutoGemmParameters.py
new file mode 100644
index 0000000..f8cb8b9
--- /dev/null
+++ b/src/library/blas/AutoGemm/AutoGemmParameters.py
@@ -0,0 +1,149 @@
+import copy
+import KernelParameters
+
+################################################################################
+# Tile Parameters for Kernel Selection Data
+################################################################################
+
+kernelSelectionData = {
+# [ size, fallback tile, [ valid tiles ] ],
+ "s":[
+ [ 4000, [ 16, 16, 6, 6], [ [ 16, 16, 6, 6] ] ],
+ [ 2496, [ 16, 16, 4, 4], [ [ 16, 16, 6, 6], [ 16, 16, 4, 4] ] ],
+ [ 2448, [ 16, 16, 6, 6], [ [ 16, 16, 6, 6] ] ],
+ [ 1600, [ 16, 16, 6, 6], [ [ 16, 16, 6, 6], [ 16, 16, 4, 4], [ 16, 16, 5, 5] ] ],
+ [ 1008, [ 16, 16, 6, 6], [ [ 16, 16, 6, 6], [ 16, 16, 4, 4], [ 16, 16, 5, 5], [ 16, 16, 3, 3] ] ],
+ [ 960, [ 16, 16, 2, 2], [ [ 16, 16, 4, 4], [ 16, 16, 3, 3], [ 16, 16, 5, 5], [ 16, 16, 2, 2] ] ],
+ [ 896, [ 16, 16, 2, 2], [ [ 16, 16, 4, 4], [ 16, 16, 6, 6], [ 16, 16, 3, 3], [ 16, 16, 5, 5], [ 16, 16, 2, 2] ] ],
+ [ 864, [ 16, 16, 2, 2], [ [ 16, 16, 6, 6], [ 16, 16, 3, 3], [ 16, 16, 5, 5], [ 16, 16, 4, 4], [ 16, 16, 2, 2] ] ],
+ [ 784, [ 16, 16, 2, 2], [ [ 16, 16, 3, 3], [ 16, 16, 5, 5], [ 16, 16, 4, 4], [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 768, [ 16, 16, 2, 2], [ [ 16, 16, 3, 3], [ 16, 16, 5, 5], [ 16, 16, 4, 4], [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 720, [ 16, 16, 2, 2], [ [ 16, 16, 4, 4], [ 16, 16, 5, 5], [ 16, 16, 4, 4], [ 16, 16, 6, 6], [ 16, 16, 3, 3] ] ],
+ [ 464, [ 16, 16, 3, 3], [ [ 16, 16, 3, 3], [ 16, 16, 4, 4], [ 16, 16, 2, 2], [ 16, 16, 5, 5] ] ],
+ [ 304, [ 16, 16, 2, 2], [ [ 16, 16, 3, 3], [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 0, [ 16, 16, 1, 1], [ [ 16, 16, 1, 1] ] ],
+ ],
+ "d":[
+ [ 5408, [ 8, 8, 6, 6], [ [ 8, 8, 6, 6], [ 16, 16, 4, 4] ] ],
+ [ 2800, [ 16, 16, 4, 4], [ [ 8, 8, 6, 6], [ 16, 16, 4, 4] ] ],
+ [ 1536, [ 16, 16, 4, 4], [ [ 8, 8, 6, 6], [ 16, 16, 4, 4], [ 16, 16, 5, 5] ] ],
+ [ 1136, [ 16, 16, 4, 4], [ [ 8, 8, 6, 6], [ 16, 16, 4, 4], [ 16, 16, 5, 5], [ 16, 16, 2, 2] ] ],
+ [ 576, [ 16, 16, 2, 2], [ [ 16, 16, 4, 4], [ 8, 8, 6, 6], [ 16, 16, 5, 5], [ 16, 16, 2, 2] ] ],
+ [ 384, [ 16, 16, 2, 2], [ [ 16, 16, 4, 4], [ 8, 8, 6, 6], [ 16, 16, 5, 5], [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 256, [ 16, 16, 1, 1], [ [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 0, [ 16, 16, 1, 1], [ [ 16, 16, 1, 1] ] ],
+ ],
+ "c":[
+ [ 3840, [ 16, 16, 4, 4], [ [ 16, 16, 4, 4] ] ],
+ [ 2592, [ 16, 16, 4, 4], [ [ 16, 16, 4, 4], [ 16, 16, 6, 6], [ 16, 16, 3, 3] ] ],
+ [ 2224, [ 16, 16, 4, 4], [ [ 16, 16, 4, 4], [ 16, 16, 3, 3], [ 16, 16, 2, 2] ] ],
+ [ 720, [ 16, 16, 2, 2], [ [ 16, 16, 4, 4], [ 16, 16, 3, 3], [ 16, 16, 2, 2], [ 16, 16, 5, 5] ] ],
+ [ 432, [ 16, 16, 2, 2], [ [ 16, 16, 2, 2], [ 16, 16, 3, 3], [ 16, 16, 1, 1] ] ],
+ [ 288, [ 16, 16, 1, 1], [ [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 0, [ 16, 16, 1, 1], [ [ 16, 16, 1, 1] ] ],
+ ],
+ "z":[
+ [ 3008, [ 16, 16, 3, 3], [ [ 16, 16, 3, 3] ] ],
+ [ 1344, [ 16, 16, 3, 3], [ [ 16, 16, 3, 3], [ 16, 16, 4, 4] ] ],
+ [ 1040, [ 16, 16, 3, 3], [ [ 16, 16, 3, 3], [ 16, 16, 4, 4], [ 16, 16, 2, 2] ] ],
+ [ 832, [ 16, 16, 2, 2], [ [ 16, 16, 3, 3], [ 16, 16, 4, 4], [ 16, 16, 2, 2] ] ],
+ [ 544, [ 16, 16, 2, 2], [ [ 16, 16, 3, 3], [ 16, 16, 2, 2] ] ],
+ [ 336, [ 16, 16, 2, 2], [ [ 16, 16, 3, 3], [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 192, [ 16, 16, 1, 1], [ [ 16, 16, 2, 2], [ 16, 16, 1, 1] ] ],
+ [ 0, [ 16, 16, 1, 1], [ [ 16, 16, 1, 1] ] ],
+ ],
+ }
+
+"""
+for testing all micro-tile sizes
+ [ 128, [ 16, 16, 8, 8], [ [ 16, 16, 8, 8] ] ],
+ [ 112, [ 16, 16, 7, 7], [ [ 16, 16, 7, 7] ] ],
+ [ 96, [ 16, 16, 6, 6], [ [ 16, 16, 6, 6] ] ],
+ [ 80, [ 16, 16, 5, 5], [ [ 16, 16, 5, 5] ] ],
+ [ 64, [ 16, 16, 4, 4], [ [ 16, 16, 4, 4] ] ],
+ [ 48, [ 16, 16, 3, 3], [ [ 16, 16, 3, 3] ] ],
+ [ 32, [ 16, 16, 2, 2], [ [ 16, 16, 2, 2] ] ],
+ [ 0, [ 16, 16, 1, 1], [ [ 16, 16, 1, 1] ] ],
+"""
+
+################################################################################
+# Non-Tile Parameters
+################################################################################
+precisions = ["s", "d", "c", "z"]
+
+orders = [ "clblasColumnMajor" ]
+
+transposes = { "s":["N", "T"], "d":["N", "T"], \
+ "c":["N", "T", "C"], "z":["N", "T", "C"] }
+
+unrolls = { "s":[16, 1], "d":[8, 1], "c":[8, 1], "z":[8, 1] }
+
+betas = [ 0, 1 ]
+
+def getTilesForPrecision(precision):
+ # valid tiles for this precision
+ tiles = []
+ tile = KernelParameters.TileParameters()
+ for sizeData in kernelSelectionData[precision]:
+ fallbackTile = sizeData[1]
+ validTiles = sizeData[2]
+ # add valid tiles
+ for tileParams in validTiles:
+ #print tileParams
+ tile.workGroupNumRows = tileParams[0]
+ tile.workGroupNumCols = tileParams[1]
+ tile.microTileNumRows = tileParams[2]
+ tile.microTileNumCols = tileParams[3]
+ tile.macroTileNumRows = tile.workGroupNumRows*tile.microTileNumRows
+ tile.macroTileNumCols = tile.workGroupNumCols*tile.microTileNumCols
+ #print tile.getName()
+ for unroll in unrolls[precision]:
+ tile.unroll = unroll
+ if tile.isValid():
+ tiles.append( copy.copy(tile) )
+ else:
+ print tile.getName() + " - SKIPPING - "
+
+ # add fallback tile
+ tile.workGroupNumRows = fallbackTile[0]
+ tile.workGroupNumCols = fallbackTile[1]
+ tile.microTileNumRows = fallbackTile[2]
+ tile.microTileNumCols = fallbackTile[3]
+ tile.macroTileNumRows = tile.workGroupNumRows*tile.microTileNumRows
+ tile.macroTileNumCols = tile.workGroupNumCols*tile.microTileNumCols
+ for unroll in unrolls[precision]:
+ tile.unroll = unroll
+ if tile.isValid():
+ tiles.append( copy.copy(tile) )
+ else:
+ print tile.getName() + " - SKIPPING - "
+
+ setTiles = set(tiles)
+ tiles = list( setTiles )
+ tiles.sort()
+ return tiles
+
+def getTransposeChoices():
+ singleTransposes = []
+ for precision in precisions:
+ for transpose in transposes[precision]:
+ singleTransposes.append( transpose )
+ singleTransposeSet = set(singleTransposes)
+ singleTranspose = list( singleTransposeSet)
+ transposeChoices = []
+ for transA in singleTranspose:
+ for transB in singleTranspose:
+ transposePair = transA+transB
+ if transposePair not in transposeChoices:
+ transposeChoices.append(transposePair)
+ return transposeChoices
+
+def getTileChoices():
+ tileChoices = []
+ for precision in precisions:
+ tilesForPrecision = getTilesForPrecision(precision)
+ for t in tilesForPrecision:
+ tile = str(t.workGroupNumRows*t.microTileNumRows) + "x" + str(t.workGroupNumCols*t.microTileNumCols)
+ if tile not in tileChoices:
+ tileChoices.append(tile)
+ return tileChoices
diff --git a/src/library/blas/AutoGemm/AutoGemmTools/AutoGemmPreCompileKernels.cpp b/src/library/blas/AutoGemm/AutoGemmTools/AutoGemmPreCompileKernels.cpp
new file mode 100644
index 0000000..1d75dbb
--- /dev/null
+++ b/src/library/blas/AutoGemm/AutoGemmTools/AutoGemmPreCompileKernels.cpp
@@ -0,0 +1,924 @@
+/* ************************************************************************
+* Copyright 2013 Advanced Micro Devices, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+* ************************************************************************/
+
+#include <assert.h>
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <stdlib.h>
+#include <string.h>
+#include <vector>
+#include <cstring>
+
+#ifdef __GNUC__
+// Linux
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/time.h>
+#else
+// Windows
+#include <Windows.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#define stat _stat
+#endif
+
+#include "CL/opencl.h"
+//#include "naive_blas.cpp"
+//using namespace NaiveBlas;
+#include "AutoGemmIncludes/AutoGemmKernelsToPreCompile.h"
+#include "AutoGemmIncludes/AutoGemmKernelSelectionSpecific.h"
+#include "UserGemmKernelSources/UserGemmClKernels.h"
+#include "UserGemmKernelSources/UserGemmKernelSourceIncludes.h"
+
+unsigned int totalKernelsToCompile;
+unsigned int numKernelsCompiled;
+char *path;
+std::ofstream includeFile;
+
+//std::clock_t clockStart;
+unsigned long long clockStart;
+unsigned long long clockFrequency;
+
+
+
+/******************************************************************************
+ * Check OpenCL Errors
+ *****************************************************************************/
+#define CL_CHECK(STATUS) \
+ if(STATUS != CL_SUCCESS) { \
+ printf("OpenCL error %i on line %u\n", STATUS, __LINE__); \
+ assert(false); \
+ }
+
+/******************************************************************************
+ * Get AMD Platform
+ *****************************************************************************/
+cl_int getAMDPlatform(cl_platform_id *platform) {
+ *platform = NULL;
+ cl_int status = CL_SUCCESS;
+
+ // get num platforms
+ cl_uint numPlatforms;
+ status = clGetPlatformIDs(0, NULL, &numPlatforms);
+ if(status != CL_SUCCESS) {
+ std::cout << "Error: clGetPlatformIDs failed. Error code: " << status << std::endl;
+ return status;
+ }
+
+ if (numPlatforms > 0) {
+ // Get selected platform
+ cl_platform_id* platforms = new cl_platform_id[numPlatforms];
+ status = clGetPlatformIDs(numPlatforms, platforms, NULL);
+ if(status != CL_SUCCESS) {
+ std::cout<<"Error: clGetPlatformIDs failed. Error code : " << status << std::endl;
+ return status;
+ }
+
+ // Print all platforms
+ for (unsigned i = 0; i < numPlatforms; ++i) {
+ char pbuf[100];
+ status = clGetPlatformInfo(platforms[i],
+ CL_PLATFORM_VENDOR,
+ sizeof(pbuf),
+ pbuf,
+ NULL);
+
+ if(status != CL_SUCCESS) {
+ std::cout<<"Error: clGetPlatformInfo failed. Error code : " << status << std::endl;
+ return status;
+ }
+
+ //std::cout << "Platform " << i << " : " << pbuf << std::endl;
+ }
+
+ // Get AMD platform
+ for (unsigned i = 0; i < numPlatforms; ++i) {
+ char pbuf[100];
+ status = clGetPlatformInfo(platforms[i],
+ CL_PLATFORM_VENDOR,
+ sizeof(pbuf),
+ pbuf,
+ NULL);
+
+ if(status != CL_SUCCESS) {
+ std::cout << "Error: clGetPlatformInfo failed. Error code: " << status << std::endl;
+ return status;
+ }
+
+ *platform = platforms[i];
+ if (!strcmp(pbuf, "Advanced Micro Devices, Inc.")) {
+ break;
+ }
+ }
+
+ // verify AMD platform
+ char pbuf[100];
+ status = clGetPlatformInfo(*platform,
+ CL_PLATFORM_VENDOR,
+ sizeof(pbuf),
+ pbuf,
+ NULL);
+
+ if(status != CL_SUCCESS) {
+ std::cout<<"Error: clGetPlatformInfo failed. Error code: " << status << std::endl;
+ return status;
+ }
+ if (strcmp(pbuf, "Advanced Micro Devices, Inc.")) {
+ std::cout << "AMD platform not found" << std::endl;
+ return CL_INVALID_PLATFORM;
+ }
+
+ } else {
+ std::cout << "No OpenCL platforms found." << std::endl;
+ return CL_INVALID_PLATFORM;
+ }
+
+ return status;
+}
+
+
+/******************************************************************************
+ * Precision -> char
+ *****************************************************************************/
+template<typename Precision> char getPrecisionChar();
+template<> char getPrecisionChar<float>(){ return 's'; }
+template<> char getPrecisionChar<double>(){ return 'd'; }
+template<> char getPrecisionChar<FloatComplex>(){ return 'c'; }
+template<> char getPrecisionChar<DoubleComplex>(){ return 'z'; }
+
+
+/******************************************************************************
+ * get kernel name
+ *****************************************************************************/
+template<typename Precision>
+int getKernelName(
+ char **kernelName,
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ bool beta,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll,
+ bool extraRow,
+ bool extraCol,
+ char *appendstring)
+{
+ int n = sprintf( *kernelName,
+ "%cgemm_%s_%s%s_B%i_M%c%03u_N%c%03u_KX%02u",
+ getPrecisionChar<Precision>(),
+ order==clblasColumnMajor ? "Col" : "Row",
+ transA==clblasNoTrans ? "N" : transA==clblasTrans ? "T" : "C",
+ transB==clblasNoTrans ? "N" : transB==clblasTrans ? "T" : "C",
+ beta ? 1 : 0,
+ extraRow ? 'L' : 'X',
+ macroTileNumRows,
+ extraCol ? 'L' : 'X',
+ macroTileNumCols,
+ unroll );
+ int n2 = 0;
+ if (appendstring != NULL)
+ {
+ n2 = sprintf((*kernelName) + n, appendstring);
+ }
+ return n2 + n;
+}
+
+template<typename Precision>
+int getStringName(
+ char **stringName,
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ bool beta,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll,
+ bool extraRow,
+ bool extraCol,
+ char *appendstring)
+{
+ int n = getKernelName<Precision>(stringName, order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, extraRow, extraCol, appendstring);
+ int n2 = sprintf( (*stringName)+n, "_bin" );
+ return n+n2;
+}
+
+template<typename Precision>
+int getFileName(
+ char **fileName,
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ bool beta,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll,
+ bool extraRow,
+ bool extraCol,
+ char *appendstring)
+{
+ int n = getKernelName<Precision>(fileName, order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, extraRow, extraCol, appendstring);
+ int n2 = sprintf( (*fileName)+n, "_bin.cpp" );
+ return n+n2;
+}
+
+template<typename Precision>
+int getPreprocessorName(
+ char **preprocessorName,
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ bool beta,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll,
+ bool extraRow,
+ bool extraCol,
+ char *appendstring)
+{
+ char kernelNameArray[64];
+ char *kernelName = kernelNameArray;
+ int n = getKernelName<Precision>(&kernelName, order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, extraRow, extraCol, appendstring);
+ for ( int i = 0; i < n; i++) {
+ kernelName[i] = toupper(kernelName[i]);
+ }
+ int n2 = sprintf( *preprocessorName, "KERNEL_%s_BIN_CPP", kernelName );
+ return n2;
+}
+
+
+/******************************************************************************
+ * get kernel binary from source
+ *****************************************************************************/
+cl_int getKernelBinaryFromSource(
+ cl_context context,
+ const char *source,
+ const char *buildOptions,
+ char **binary,
+ size_t *binarySize)
+{
+ cl_int status = CL_SUCCESS;
+
+ // create program
+ cl_program program = clCreateProgramWithSource(context,1, &source, NULL, &status);
+ CL_CHECK(status);
+
+ cl_uint numDevicesInContext;
+ status = clGetContextInfo(context, CL_CONTEXT_NUM_DEVICES, sizeof(cl_uint), &numDevicesInContext, NULL);
+ CL_CHECK(status);
+
+ // get devices
+ //printf("Devices: %u\n", numDevicesInContext);
+ cl_device_id* devices = new cl_device_id[numDevicesInContext];
+ clGetContextInfo(context, CL_CONTEXT_DEVICES, numDevicesInContext*sizeof(cl_device_id), devices, NULL);
+ CL_CHECK(status);
+
+ // choose device 0
+ cl_device_id device = devices[0];
+
+ // build program for device
+ status = clBuildProgram(program, 1, &device, buildOptions, NULL, NULL);
+
+
+ // print build failure
+ if (status != CL_SUCCESS) {
+ printf("clBuildProgram Failed\n");
+ printf("status = %d\n", status);
+
+ size_t len=0;
+ clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &len);
+ char* buildLog = new char[len];
+
+ printf("Error: Failed to build program executable!\n");
+ clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, len*sizeof(char), buildLog, 0);
+ printf("\nBuild Log:\n\n");
+ printf("%s\n", buildLog);
+ printf("\n\nKernel String:\n\n");
+ printf("%s\n", source);
+
+ binary[0] = 0;
+ *binarySize = 0;
+ return status;
+ }
+
+
+ // get binary from program
+ status = clGetProgramInfo(program, CL_PROGRAM_BINARY_SIZES, sizeof(size_t), binarySize, NULL);
+ //printf("BinarySize: %llu\n", *binarySize);
+ binary[0] = new char[*binarySize];
+ //for (int i = 0; i < *binarySize; i++) binary[0][i] = 512;
+
+ //size_t size = 2;
+ //status = -1;
+ //for (int i = 0; status; i++) {
+ // printf("size=%i\n", i);
+ status = clGetProgramInfo(program, CL_PROGRAM_BINARIES, 8 /*?*/, binary, NULL);
+ //}
+ CL_CHECK(status);
+
+ //for (int i = 0; i < *binarySize; i++) {
+ // std::cout << std::setw(3) << (int)binary[0][i] << ",";
+ //}
+ //printf("binary[0][0] = %p\n", binary[0][0]);
+ //printf("binary[0] = %p\n", binary[0]);
+ //printf("binary = %p\n", binary);
+ //printf("&binary = %p\n", &binary);
+
+ return CL_SUCCESS;
+}
+
+
+/******************************************************************************
+ * write binary to stream
+ *****************************************************************************/
+void writeBinaryToStream(std::ostream & out, char *binary, size_t binarySize) {
+ for(int i = 0; i < binarySize; i++) {
+
+ out << std::setw(4) << (int) binary[i];
+
+ if(i < binarySize-1) {
+ out << ",";
+ }
+ if((i+1)%16 == 0) {
+ out << std::endl;
+ }
+ }
+ out << std::endl;
+}
+
+
+/******************************************************************************
+ * Pre-compile kernels within parameter group and write to file
+ *****************************************************************************/
+template<typename Precision>
+void compileKernelAndWriteToFile(
+ cl_context context,
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ bool beta,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll,
+ bool extraRow,
+ bool extraCol,
+ const char *source,
+ const char *buildOptions,
+ char* appendString)
+{
+ // get kernel name
+ char stringNameArray[64];
+ char fileNameArray[64];
+ char preprocessorNameArray[64];
+ char *stringName = &stringNameArray[0];
+ char *fileName = &fileNameArray[0];
+ char *preprocessorName = &preprocessorNameArray[0];
+ int stringNameLength = getStringName<Precision>(&stringName, order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, extraRow, extraCol, appendString);
+ int fileNameLength = getFileName<Precision>(&fileName, order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, extraRow, extraCol, appendString);
+ int preprocessorNameLength = getPreprocessorName<Precision>(&preprocessorName, order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, extraRow, extraCol, appendString);
+
+ // get kernel binary
+ char **kernelBinary = new char*[1];
+ kernelBinary[0] = 0;
+ size_t kernelBinarySize;
+ cl_int status = getKernelBinaryFromSource(context, source, buildOptions, kernelBinary, &kernelBinarySize);
+
+ if (status == CL_SUCCESS) {
+ // write binary to file
+ std::ofstream kernelFile;
+ std::string fullFilePath;
+ fullFilePath += path;
+ fullFilePath += fileName;
+ kernelFile.open(fullFilePath.c_str(), std::ios::out);
+ kernelFile << "/* AutoGemm Pre-Compiled kernel binary */" << std::endl << std::endl;
+ kernelFile << "#define " << preprocessorName << std::endl << std::endl;
+
+ kernelFile << "char " << stringName << "Array[" << kernelBinarySize << "] = {" << std::endl;
+ //kernelFile << "unsigned char *" << stringName << " = {" << std::endl;
+ //kernelFile << "unsigned char " << stringName << "[] = {" << std::endl;
+
+ writeBinaryToStream( kernelFile, *kernelBinary, kernelBinarySize );
+ kernelFile << "};" << std::endl;
+ kernelFile << "unsigned char *" << stringName << " = " << "reinterpret_cast<unsigned char *>(" << stringName << "Array);" << std::endl;
+ kernelFile << "size_t " << stringName << "Size = " << kernelBinarySize << ";" << std::endl;
+ kernelFile.close();
+
+ // add file to include
+ includeFile << "#include \"AutoGemmKernelBinaries/" << fileName << "\"" << std::endl;
+ }
+ if (kernelBinary[0])
+ delete[] kernelBinary[0];
+ delete[] kernelBinary;
+
+ // report kernel compiled
+ numKernelsCompiled++;
+
+ // how much time left
+ unsigned long long clockCurrent;
+ #if defined( _WIN32 )
+ ::QueryPerformanceCounter( reinterpret_cast<LARGE_INTEGER*>( &clockCurrent ) );
+#else
+ struct timeval s;
+ gettimeofday(&s, 0);
+ clockCurrent = (unsigned long long)s.tv_sec * 1000000 + (unsigned long long)s.tv_usec;
+#endif
+
+
+
+ double elapsedTimeSec = ((double) clockCurrent - clockStart) / clockFrequency;
+ double timePerKernel = elapsedTimeSec / numKernelsCompiled;
+ double timeRemaining = timePerKernel * (totalKernelsToCompile - numKernelsCompiled);
+ //printf("AutoGemm-PreCompile[%3u/%3u]: %s %7u bytes ( %.0f sec remaining)\n", numKernelsCompiled, totalKernelsToCompile, stringName, kernelBinarySize, timeRemaining);
+ std::cout << "AutoGemm-PreCompile[" << std::setw(3) << numKernelsCompiled << "/" << std::setw(3) << totalKernelsToCompile << "]: " << stringName << std::setw(4) << kernelBinarySize/1024 << " kB (" << std::setw(4) << (int) timeRemaining << " sec remaining)" << std::endl;
+}
+
+
+/******************************************************************************
+ * compile kernels within parameter group and write to file
+ *****************************************************************************/
+template<typename Precision>
+cl_int compileKernelGroupAndWriteToFile(
+ cl_context context,
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ bool beta,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll )
+{
+ const char *tileKernelSource;
+ const char *rowKernelSource;
+ const char *colKernelSource;
+ const char *cornerKernelSource;
+ const char *sourceBuildOptions;
+ const unsigned char *tileKernelBinary;
+ const unsigned char *rowKernelBinary;
+ const unsigned char *colKernelBinary;
+ const unsigned char *cornerKernelBinary;
+ size_t *tileKernelBinarySize;
+ size_t *rowKernelBinarySize;
+ size_t *colKernelBinarySize;
+ size_t *cornerKernelBinarySize;
+ const char *binaryBuildOptions;
+ cl_kernel *tileClKernel;
+ cl_kernel *rowClKernel;
+ cl_kernel *colClKernel;
+ cl_kernel *cornerClKernel;
+ unsigned int workGroupNumRows;
+ unsigned int workGroupNumCols;
+ unsigned int microTileNumRows;
+ unsigned int microTileNumCols;
+ bool kernelFound = gemmSelectKernelSpecific<Precision>(
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ &tileKernelSource,
+ &rowKernelSource,
+ &colKernelSource,
+ &cornerKernelSource,
+ &sourceBuildOptions,
+ &tileKernelBinary,
+ &rowKernelBinary,
+ &colKernelBinary,
+ &cornerKernelBinary,
+ &tileKernelBinarySize,
+ &rowKernelBinarySize,
+ &colKernelBinarySize,
+ &cornerKernelBinarySize,
+ &binaryBuildOptions,
+ &tileClKernel,
+ &rowClKernel,
+ &colClKernel,
+ &cornerClKernel,
+ &workGroupNumRows,
+ &workGroupNumCols,
+ µTileNumRows,
+ µTileNumCols );
+
+ if (!kernelFound) {
+ totalKernelsToCompile -= 4;
+ char stringNameArray[64];
+ char *stringName = &stringNameArray[0];
+ int stringNameLength = getStringName<Precision>( &stringName, order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, 0, 0, NULL);
+ printf("AutoGemm-PreCompile: %s not found; skipping.\n", stringName );
+ return 0;
+ }
+
+ compileKernelAndWriteToFile<Precision>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ false, // extra row
+ false, // extra col
+ tileKernelSource,
+ sourceBuildOptions,
+ NULL);
+
+ compileKernelAndWriteToFile<Precision>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ true, // extra row
+ false, // extra col
+ rowKernelSource,
+ sourceBuildOptions,
+ NULL);
+
+ compileKernelAndWriteToFile<Precision>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ false, // extra row
+ true, // extra col
+ colKernelSource,
+ sourceBuildOptions,
+ NULL);
+
+ compileKernelAndWriteToFile<Precision>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ true, // extra row
+ true, // extra col
+ cornerKernelSource,
+ sourceBuildOptions,
+ NULL);
+
+ return 1;
+}
+
+/******************************************************************************
+ * Main
+ *****************************************************************************/
+int main( int argc, char *argv[] ) {
+
+ if (argc < 2 ) {
+ printf("AutoGemmPreCompileKernels requires output path as argument\n");
+ exit(-1);
+ }
+
+ size_t pathLength = std::strlen(argv[1]);
+ path = new char[pathLength+64];
+ sprintf(path, "%s/include/AutoGemmKernelBinaries/", argv[1]);
+ //printf("AutoGemm-PreCompile: writing to %s\n", path);
+
+ std::string fullIncludeFilePath;
+ fullIncludeFilePath += path;
+ fullIncludeFilePath += "AutoGemmKernelBinariesPreCompiled.h";
+ includeFile.open(fullIncludeFilePath.c_str(), std::ios::out);
+ const char *includeFileHeader =
+ "/*****************************************************************************\n"
+ " * this file auto-generated by AutoGemmPreCompileKernels\n"
+ " ****************************************************************************/\n\n";
+ includeFile << includeFileHeader;
+
+ // get AMD platform
+ cl_platform_id platform;
+ cl_int status = getAMDPlatform( &platform );
+ CL_CHECK(status);
+
+ cl_uint numDevices;
+ status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 0, NULL, &numDevices);
+ CL_CHECK(status);
+
+ // get all gpu devices
+ //printf("NumDevicesInPlatform: %u\n", numDevices);
+ cl_device_id* devices = new cl_device_id[numDevices];
+ clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, numDevices, devices, NULL);
+ CL_CHECK(status);
+
+ // choose device 0
+ cl_device_id device = devices[0];
+
+ // create context
+ cl_context_properties cps[3] = {
+ CL_CONTEXT_PLATFORM,
+ (cl_context_properties)platform,
+ 0
+ };
+ cl_context context = clCreateContext(
+ cps,
+ 1, // device
+ &device,
+ NULL,
+ NULL,
+ &status);
+ CL_CHECK(status);
+
+ cl_uint numDevicesInContext;
+ status = clGetContextInfo(context, CL_CONTEXT_NUM_DEVICES, sizeof(cl_uint), &numDevicesInContext, NULL);
+ //printf("NumDevicesInContext: %u\n", numDevicesInContext);
+ CL_CHECK(status);
+
+
+ clblasOrder order;
+ clblasTranspose transA;
+ clblasTranspose transB;
+ bool beta;
+ unsigned int macroTileNumRows;
+ unsigned int macroTileNumCols;
+ unsigned int unroll;
+
+ // timer
+ clockStart = clock();
+
+#if defined( _WIN32 )
+ // OS call to get ticks per second2
+ ::QueryPerformanceFrequency( reinterpret_cast<LARGE_INTEGER*>( &clockFrequency ) );
+#else
+ clockFrequency = 1000000;
+#endif
+
+
+
+#if defined( _WIN32 )
+ ::QueryPerformanceCounter( reinterpret_cast<LARGE_INTEGER*>( &clockStart ) );
+#else
+ struct timeval s;
+ gettimeofday(&s, 0);
+ clockStart = (unsigned long long)s.tv_sec * 1000000 + (unsigned long long)s.tv_usec;
+#endif
+ const int specialKernelCount = user_kernel_count;
+ totalKernelsToCompile = gemmPreCompileNum;
+ totalKernelsToCompile *= 4;
+ totalKernelsToCompile += specialKernelCount;
+ numKernelsCompiled = 0;
+ //precompile user defined kernels
+ //all of the user defined special kernels will be precompiled if precompile is active
+ //there are 7 user defined special kernels refer to UserGemmKernelIncludes.h
+ const char *tileKernelSourceArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_src,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_src,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src
+ };
+ const unsigned char *tileKernelBinaryArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_bin,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_bin,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_bin,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_bin,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_bin,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_bin,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_bin
+ };
+ size_t tileKernelBinarySizeArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_binSize,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_binSize,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_binSize,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_binSize,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_binSize,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_binSize,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_binSize
+ };
+ unsigned int workGroupNumRowsArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_workGroupNumRows,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_workGroupNumRows,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_workGroupNumRows,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_workGroupNumRows,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows
+ };
+ unsigned int workGroupNumColsArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_workGroupNumCols,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_workGroupNumCols,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_workGroupNumCols,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_workGroupNumCols,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols
+ };
+ unsigned int microTileNumRowsArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_microTileNumRows,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_microTileNumRows,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_microTileNumRows,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_microTileNumRows,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_microTileNumRows,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_microTileNumRows,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_microTileNumRows
+ };
+ unsigned int microTileNumColsArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_microTileNumCols,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_microTileNumCols,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_microTileNumCols,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_microTileNumCols,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_microTileNumCols,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_microTileNumCols,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_microTileNumCols
+ };
+ unsigned int unrollArray[specialKernelCount] =
+ {
+ sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_unroll,
+ sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_unroll,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_unroll,
+ sgemm_Col_NT_B1_MX128_NX128_KX16_unroll,
+ sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_unroll,
+ sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_unroll,
+ sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_unroll
+ };
+ char *appendStringArray[specialKernelCount] =
+ {
+ "_ROW",
+ "_COLUMN",
+ "_SINGLE",
+ NULL,
+ "_BRANCH",
+ "_BRANCH",
+ "_BRANCH"
+ };
+ clblasTranspose transA_Array[specialKernelCount] =
+ {
+ clblasNoTrans,
+ clblasNoTrans,
+ clblasNoTrans,
+ clblasNoTrans,
+ clblasNoTrans,
+ clblasNoTrans,
+ clblasTrans
+ };
+ clblasTranspose transB_Array[specialKernelCount] =
+ {
+ clblasTrans,
+ clblasTrans,
+ clblasTrans,
+ clblasTrans,
+ clblasNoTrans,
+ clblasTrans,
+ clblasNoTrans
+ };
+
+ for (int i = 0; i < specialKernelCount; i++)
+ {
+ const char *tileKernelSource;
+ const unsigned char *tileKernelBinary;
+ size_t tileKernelBinarySize;
+ const char *binaryBuildOptions;
+ //cl_kernel *tileClKernel;
+ unsigned int workGroupNumRows;
+ unsigned int workGroupNumCols;
+ unsigned int unroll;
+
+ tileKernelSource = tileKernelSourceArray[i];
+ tileKernelBinary = tileKernelBinaryArray[i];
+ tileKernelBinarySize = tileKernelBinarySizeArray[i];
+ binaryBuildOptions = User_binBuildOptions;
+ workGroupNumRows = workGroupNumRowsArray[i];
+ workGroupNumCols = workGroupNumColsArray[i];
+ macroTileNumRows = microTileNumRowsArray[i] * workGroupNumRowsArray[i];
+ macroTileNumCols = microTileNumColsArray[i] * workGroupNumColsArray[i];
+ unroll = unrollArray[i];
+ beta = 1.0;
+ char *appendString = appendStringArray[i];
+
+
+ compileKernelAndWriteToFile<float>(
+ context,
+ clblasColumnMajor,
+ transA_Array[i],
+ transB_Array[i],
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ false, // extra row
+ false, // extra col
+ tileKernelSource,
+ binaryBuildOptions,
+ appendString);
+
+ }
+
+ // for each kernel to be pre-compiled
+
+ //totalKernelsToCompile = gemmPreCompileNum;
+ //totalKernelsToCompile *= 4;
+ //numKernelsCompiled = 0;
+
+ for (unsigned int i = 0; i < gemmPreCompileNum ; i++) {
+ // unload parameters
+ // idx 0 is precision
+ order = gemmPreCompile[i][1]==1 ? clblasColumnMajor : clblasRowMajor;
+ transA = gemmPreCompile[i][2]==0 ? clblasNoTrans : gemmPreCompile[i][2]==1 ? clblasTrans : clblasConjTrans;
+ transB = gemmPreCompile[i][3]==0 ? clblasNoTrans : gemmPreCompile[i][3]==1 ? clblasTrans : clblasConjTrans;
+ beta = gemmPreCompile[i][4]==1;
+ macroTileNumRows = gemmPreCompile[i][5];
+ macroTileNumCols = gemmPreCompile[i][6];
+ unroll = gemmPreCompile[i][7];
+
+ if (gemmPreCompile[i][0] == 0) { // sgemm
+ compileKernelGroupAndWriteToFile<float>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll
+ );
+ } else if (gemmPreCompile[i][0] == 1) { // dgemm
+ compileKernelGroupAndWriteToFile<double>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll
+ );
+ } else if (gemmPreCompile[i][0] == 2) { // cgemm
+ compileKernelGroupAndWriteToFile<FloatComplex>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll
+ );
+ } else if (gemmPreCompile[i][0] == 3) { // zgemm
+ compileKernelGroupAndWriteToFile<DoubleComplex>(
+ context,
+ order,
+ transA,
+ transB,
+ beta,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll
+ );
+ }
+ }// end for
+
+ //precompile user defined kernels
+
+
+ unsigned long long clockCurrent;
+#if defined( _WIN32 )
+ ::QueryPerformanceCounter( reinterpret_cast<LARGE_INTEGER*>( &clockCurrent ) );
+#else
+ gettimeofday(&s, 0);
+ clockCurrent = (unsigned long long)s.tv_sec * 1000000 + (unsigned long long)s.tv_usec;
+#endif
+ double elapsedTimeSec = ((double)clockCurrent - clockStart) / clockFrequency;
+ includeFile.close();
+ std::cout << "Total Compile Time: " << elapsedTimeSec << " sec" << std::endl;
+ //system("PAUSE");
+ return 0;
+}
+
diff --git a/src/library/blas/AutoGemm/AutoGemmTools/AutoGemmUtil.h b/src/library/blas/AutoGemm/AutoGemmTools/AutoGemmUtil.h
new file mode 100644
index 0000000..cee60b4
--- /dev/null
+++ b/src/library/blas/AutoGemm/AutoGemmTools/AutoGemmUtil.h
@@ -0,0 +1,793 @@
+/* ************************************************************************
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ************************************************************************/
+
+#if defined (_MSC_VER)
+#define __template_static static
+#define isnan(x) _isnan((x))
+#pragma warning( disable : 4290 )
+#else /* _MSC_VER */
+#define __template_static
+#endif /* !_MSC_VER */
+
+#include <clBLAS.h>
+
+
+#define CREAL(v) ((v).s[0])
+#define CIMAG(v) ((v).s[1])
+
+// Type-dependent constants
+
+template<typename T>
+static T
+ZERO()
+{
+ return static_cast<T>(0.0);
+}
+
+template<>
+__template_static FloatComplex
+ZERO<FloatComplex>()
+{
+ return floatComplex(0.0, 0.0);
+}
+
+template<>
+__template_static DoubleComplex
+ZERO<DoubleComplex>()
+{
+ return doubleComplex(0.0, 0.0);
+}
+
+template<class T>
+static T
+ONE()
+{
+ return static_cast<T>(1.0);
+}
+
+template<>
+__template_static FloatComplex
+ONE<FloatComplex>()
+{
+ return floatComplex(1.0, 0.0);
+}
+
+template<>
+__template_static DoubleComplex
+ONE<DoubleComplex>()
+{
+ return doubleComplex(1.0, 0.0);
+}
+
+template<class T>
+static T
+TWO()
+{
+ return static_cast<T>(2.0);
+}
+
+template<>
+__template_static FloatComplex
+TWO<FloatComplex>()
+{
+ return floatComplex(2.0, 0.0);
+}
+
+template<>
+__template_static DoubleComplex
+TWO<DoubleComplex>()
+{
+ return doubleComplex(2.0, 0.0);
+}
+
+template<class T>
+static bool
+isNAN(T x)
+{
+ return (isnan(x) != 0);
+}
+
+template<>
+__template_static bool
+isNAN(FloatComplex x)
+{
+ return (isNAN(CREAL(x)) && isNAN(CIMAG(x)));
+}
+
+template<>
+__template_static bool
+isNAN(DoubleComplex x)
+{
+ return (isNAN(CREAL(x)) && isNAN(CIMAG(x)));
+}
+
+/* Type-dependent random() */
+
+template<class T>
+static T
+random(cl_double limit)
+{
+ T v;
+ cl_ulong l = static_cast<cl_ulong>(limit);
+
+ if (l == 0) {
+ return 0;
+ }
+ v = static_cast<float>(rand() % l);
+ if ((rand() % 2) == 1)
+ v = -v;
+ return v;
+}
+
+template<typename T>
+static T
+random(cl_double left, cl_double right)
+{
+ T v;
+ T l = static_cast<T>(left);
+
+ v = random<T>(right - left);
+ if (v < 0) {
+ v -= l;
+ }
+ else {
+ v += l;
+ }
+ return v;
+}
+
+template<class T>
+static T
+random()
+{
+ return random<T>(static_cast<T>(10));
+}
+
+template<>
+__template_static FloatComplex
+random<FloatComplex>()
+{
+ return floatComplex(random<cl_float>(), random<cl_float>());
+}
+
+template<>
+__template_static FloatComplex
+random<FloatComplex>(cl_double limit)
+{
+ return floatComplex(random<cl_float>(limit), random<cl_float>(limit));
+}
+
+template<>
+__template_static FloatComplex
+random<FloatComplex>(cl_double left, cl_double right)
+{
+ return floatComplex(random<cl_float>(left, right), random<cl_float>(left, right));
+}
+
+
+template<>
+__template_static DoubleComplex
+random<DoubleComplex>()
+{
+ return doubleComplex(random<cl_double>(), random<cl_double>());
+}
+
+template<>
+__template_static DoubleComplex
+random<DoubleComplex>(cl_double limit)
+{
+ return doubleComplex(random<cl_double>(limit), random<cl_double>(limit));
+}
+
+template<>
+__template_static DoubleComplex
+random<DoubleComplex>(cl_double left, cl_double right)
+{
+ return doubleComplex(random<cl_double>(left, right), random<cl_double>(left, right));
+}
+
+/* Boolean operators */
+
+template<class T>
+static bool
+operator==(T a, T b)
+{
+ return (a == b);
+}
+
+template<>
+__template_static bool
+operator==(FloatComplex a, FloatComplex b)
+{
+ return ((CREAL(a) == CREAL(b)) && (CIMAG(a) == CIMAG(b)));
+}
+
+template<>
+__template_static bool
+operator==(DoubleComplex a, DoubleComplex b)
+{
+ return ((CREAL(a) == CREAL(b)) && (CIMAG(a) == CIMAG(b)));
+}
+
+template<class T>
+static bool
+operator!=(T a, T b)
+{
+ return !(a == b);
+}
+
+/* math operators */
+
+static __inline
+float conjugate(float elem)
+{
+ return elem;
+}
+
+static __inline
+double conjugate(double elem)
+{
+ return elem;
+}
+
+static __inline
+FloatComplex conjugate(FloatComplex elem)
+{
+ return floatComplex(CREAL(elem), -CIMAG(elem));
+}
+
+static __inline
+DoubleComplex conjugate(DoubleComplex elem)
+{
+ return doubleComplex(CREAL(elem), -CIMAG(elem));
+}
+
+static __inline FloatComplex
+operator+(FloatComplex a, FloatComplex b)
+{
+ return floatComplex(CREAL(a) + CREAL(b), CIMAG(a) + CIMAG(b));
+}
+
+static __inline FloatComplex
+operator-(FloatComplex a, FloatComplex b)
+{
+ return floatComplex(CREAL(a) - CREAL(b), CIMAG(a) - CIMAG(b));
+}
+
+static __inline FloatComplex
+operator*(FloatComplex a, FloatComplex b)
+{
+ return floatComplex(
+ CREAL(a) * CREAL(b) - CIMAG(a) * CIMAG(b),
+ CREAL(a) * CIMAG(b) + CREAL(b) * CIMAG(a));
+}
+
+static __inline FloatComplex
+operator*(FloatComplex a, cl_float b)
+{
+ return floatComplex(CREAL(a) * b, CIMAG(a) * b);
+}
+
+static __inline FloatComplex
+operator/(FloatComplex a, FloatComplex b)
+{
+ cl_float div = CREAL(b) * CREAL(b) + CIMAG(b) * CIMAG(b);
+
+ return floatComplex(
+ (CREAL(a) * CREAL(b) + CIMAG(a) * CIMAG(b)) / div,
+ (CREAL(b) * CIMAG(a) - CREAL(a) * CIMAG(b)) / div);
+}
+
+static __inline FloatComplex
+operator/(FloatComplex a, cl_float b)
+{
+ return floatComplex(CREAL(a) / b, CIMAG(a) / b);
+}
+
+static __inline DoubleComplex
+operator+(DoubleComplex a, DoubleComplex b)
+{
+ return doubleComplex(CREAL(a) + CREAL(b), CIMAG(a) + CIMAG(b));
+}
+
+static __inline DoubleComplex
+operator-(DoubleComplex a, DoubleComplex b)
+{
+ return doubleComplex(CREAL(a) - CREAL(b), CIMAG(a) - CIMAG(b));
+}
+
+static __inline DoubleComplex
+operator*(DoubleComplex a, DoubleComplex b)
+{
+ return doubleComplex(
+ CREAL(a) * CREAL(b) - CIMAG(a) * CIMAG(b),
+ CREAL(a) * CIMAG(b) + CREAL(b) * CIMAG(a));
+}
+
+static __inline DoubleComplex
+operator*(DoubleComplex a, cl_double b)
+{
+ return doubleComplex(CREAL(a) * b, CIMAG(a) * b);
+}
+
+static __inline DoubleComplex
+operator/(DoubleComplex a, DoubleComplex b)
+{
+ cl_double div = CREAL(b) * CREAL(b) + CIMAG(b) * CIMAG(b);
+
+ return doubleComplex(
+ (CREAL(a) * CREAL(b) + CIMAG(a) * CIMAG(b)) / div,
+ (CREAL(b) * CIMAG(a) - CREAL(a) * CIMAG(b)) / div);
+}
+
+static __inline DoubleComplex
+operator/(DoubleComplex a, cl_double b)
+{
+ return doubleComplex(CREAL(a) / b, CIMAG(a) / b);
+}
+
+cl_int
+module(cl_int a)
+{
+ return abs(a);
+}
+
+cl_float
+module(cl_float a)
+{
+ return fabsf(a);
+}
+
+cl_double
+module(cl_double a)
+{
+ return fabs(a);
+}
+cl_float
+module(FloatComplex a)
+{
+ if ((CREAL(a) == 0.0) && (CIMAG(a) == 0.0))
+ return 0.0;
+ return sqrtf(CREAL(a) * CREAL(a) + CIMAG(a) * CIMAG(a));
+}
+
+cl_double
+module(DoubleComplex a)
+{
+ if ((CREAL(a) == 0.0) && (CIMAG(a) == 0.0))
+ return 0.0;
+ return sqrt(CREAL(a) * CREAL(a) + CIMAG(a) * CIMAG(a));
+}
+
+#define FLOAT_UPPER_BOUND pow(2.0, 23)
+#define DOUBLE_UPPER_BOUND pow(2.0, 52)
+
+// Type-dependant constants
+template <class T>
+static cl_double UPPER_BOUND();
+template<>
+__template_static cl_double UPPER_BOUND<cl_float>() { return FLOAT_UPPER_BOUND; }
+template<>
+__template_static cl_double UPPER_BOUND<cl_double>() { return DOUBLE_UPPER_BOUND;}
+template<>
+__template_static cl_double UPPER_BOUND<FloatComplex>() { return FLOAT_UPPER_BOUND; }
+template<>
+__template_static cl_double UPPER_BOUND<DoubleComplex>() { return DOUBLE_UPPER_BOUND; }
+
+/* Provide simple access to vector elements */
+
+template <typename ElemType, typename IncType> class VectorAccessor {
+public:
+ VectorAccessor(
+ ElemType *vector,
+ size_t len,
+ IncType inc,
+ bool conj=false) : vector_(vector), inc_(inc), len_(len), conj_(conj)
+ {
+ /* do nothing */
+ }
+
+ ElemType&
+ operator [] (size_t idx) throw (std::string)
+ {
+ ElemType *el;
+
+ if (idx >= len_) {
+ throw std::string("Trying to access vector beyond boundary!");
+ }
+
+ if (inc_ > 0) {
+ el = vector_ + idx * inc_;
+ }
+ else {
+ el = vector_ + (len_ - idx - 1) * (-inc_);
+ }
+
+ if (conj_) {
+ tmp_ = conjugate(*el);
+ return tmp_;
+ }
+ else {
+ return *el;
+ }
+ }
+
+private:
+ ElemType *vector_;
+ ElemType tmp_;
+ IncType inc_;
+ size_t len_;
+ bool conj_;
+};
+
+/* Mapping between logical and physical matrix layout */
+template <typename T> class MatrixAccessor {
+public:
+ MatrixAccessor(
+ T *matrix,
+ clblasOrder order,
+ clblasTranspose trans,
+ size_t nrRows,
+ size_t nrCols,
+ size_t ld) : matrix_(matrix), nrRows_(nrRows), nrCols_(nrCols), ld_(ld)
+ {
+ conj_ = (trans == clblasConjTrans);
+
+ if ((order == clblasColumnMajor && trans == clblasNoTrans) ||
+ (order == clblasRowMajor && trans != clblasNoTrans))
+ {
+ tra_ = true;
+ }
+ else {
+ tra_ = false;
+ }
+ }
+
+ void flipTransposing(void)
+ {
+ tra_ = !tra_;
+ }
+
+ VectorAccessor<T, size_t>
+ operator [] (size_t row) const throw (std::string)
+ {
+ T *vector;
+ size_t inc;
+
+ if (row >= nrRows_) {
+ throw std::string("Trying to access matrix beyond boundary!");
+ }
+
+ if (tra_) {
+ vector = matrix_ + row;
+ inc = ld_;
+ }
+ else {
+ vector = matrix_ + row * ld_;
+ inc = 1;
+ }
+
+ return VectorAccessor<T, size_t>(vector, nrCols_, inc, conj_);
+ }
+
+private:
+ T *matrix_;
+ bool tra_;
+ bool conj_;
+ size_t nrRows_;
+ size_t nrCols_;
+ size_t ld_;
+};
+
+
+template <typename T> __template_static void
+gemm(
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ size_t M,
+ size_t N,
+ size_t K,
+ T alpha,
+ const T *A,
+ size_t lda,
+ const T *B,
+ size_t ldb,
+ T beta,
+ T *C,
+ size_t ldc)
+{
+ MatrixAccessor<T> ma(const_cast<T*>(A), order, transA, M, K, lda);
+ MatrixAccessor<T> mb(const_cast<T*>(B), order, transB, K, N, ldb);
+ MatrixAccessor<T> mc(C, order, clblasNoTrans, M, N, ldc);
+ size_t i, j, k;
+ T tmp;
+
+ for (i = 0; i < M; i++) {
+ for (j = 0; j < N; j++) {
+ tmp = ZERO<T>();
+ for (k = 0; k < K; k++) {
+ tmp = tmp + ma[i][k] * mb[k][j];
+ }
+ mc[i][j] = mc[i][j] * beta + tmp * alpha;
+ }
+ }
+}
+
+template<typename T> __template_static void
+trmm(
+ clblasOrder order,
+ clblasSide side,
+ clblasUplo uplo,
+ clblasTranspose transA,
+ clblasDiag diag,
+ size_t M,
+ size_t N,
+ T alpha,
+ const T *A,
+ size_t lda,
+ T *B,
+ size_t ldb)
+{
+ size_t i, j, k;
+ size_t row, col;
+ size_t rowsA = (side == clblasLeft) ? M : N;
+ size_t colsB = (side == clblasLeft) ? N : M;
+ MatrixAccessor<T> ma(const_cast<T*>(A), order, transA, rowsA, rowsA, lda);
+ MatrixAccessor<T> mb(B, order, clblasNoTrans, rowsA, colsB, ldb);
+ T tmp, a;
+ bool revPass;
+
+ revPass = (uplo == clblasLower) ^ (transA != clblasNoTrans);
+ if (side == clblasRight) {
+ ma.flipTransposing();
+ mb.flipTransposing();
+ revPass = !revPass;
+ }
+
+ for (i = 0; i < rowsA; i++) {
+ row = (revPass) ? (rowsA - i - 1) : i;
+ for (j = 0; j < colsB; j++) {
+ size_t boundK = (revPass) ? row : (rowsA - row - 1);
+
+ tmp = ZERO<T>();
+ for (k = 0; k <= boundK; k++) {
+ col = (revPass) ? k : (rowsA - k - 1);
+ if ((k == boundK) && (diag == clblasUnit)) {
+ a = ONE<T>();
+ }
+ else {
+ a = ma[row][col];
+ }
+ tmp = tmp + a * mb[col][j];
+ }
+ mb[row][j] = tmp * alpha;
+ }
+ }
+}
+
+template<typename T> __template_static void
+trsm(
+ clblasOrder order,
+ clblasSide side,
+ clblasUplo uplo,
+ clblasTranspose transA,
+ clblasDiag diag,
+ size_t M,
+ size_t N,
+ T alpha,
+ const T *A,
+ size_t lda,
+ T *B,
+ size_t ldb)
+{
+ size_t i, j, k;
+ size_t row, col;
+ size_t rowsA = (side == clblasLeft) ? M : N;
+ size_t colsB = (side == clblasLeft) ? N : M;
+ MatrixAccessor<T> ma(const_cast<T*>(A), order, transA, rowsA, rowsA, lda);
+ MatrixAccessor<T> mb(B, order, clblasNoTrans, rowsA, colsB, ldb);
+ T tmp, a;
+ bool revPass;
+
+ revPass = (uplo == clblasUpper) ^ (transA != clblasNoTrans);
+ if (side == clblasRight) {
+ ma.flipTransposing();
+ mb.flipTransposing();
+ revPass = !revPass;
+ }
+
+ for (i = 0; i < rowsA; i++) {
+ row = (revPass) ? (rowsA - i - 1) : i;
+ for (j = 0; j < colsB; j++) {
+ size_t boundK = (revPass) ? (rowsA - row - 1) : row;
+
+ tmp = ZERO<T>();
+ for (k = 0; k <= boundK; k++) {
+ col = (revPass) ? (rowsA - k - 1) : k;
+ if (col == row) {
+ a = (diag == clblasUnit) ? ONE<T>() : ma[row][col];
+ tmp = (mb[row][j] - tmp) / a;
+ }
+ else {
+ tmp = tmp + ma[row][col] * mb[col][j];
+ }
+
+ }
+ mb[row][j] = tmp;
+ }
+ }
+
+ for (i = 0; i < rowsA; i++) {
+ for (j = 0; j < colsB; j++) {
+ mb[i][j] = mb[i][j] * alpha;
+ }
+ }
+}
+
+template <typename T> __template_static void
+syrk(
+ clblasOrder order,
+ clblasUplo uplo,
+ clblasTranspose trans,
+ size_t N,
+ size_t K,
+ T alpha,
+ const T *A,
+ size_t lda,
+ T beta,
+ T *C,
+ size_t ldc)
+{
+ size_t i, j, k;
+ clblasTranspose tr =
+ trans == clblasNoTrans ? clblasNoTrans : clblasTrans;
+ MatrixAccessor<T> ma(const_cast<T*>(A), order, tr, N, K, lda);
+ MatrixAccessor<T> mc(C, order, clblasNoTrans, N, N, ldc);
+ T tmp;
+
+ for (i = 0; i < N; i++) {
+ for (j = 0; j < N; j++) {
+ if ((uplo == clblasLower && j > i) ||
+ (uplo == clblasUpper && i > j)) {
+ continue;
+ }
+
+ tmp = ZERO<T>();
+ for (k = 0; k < K; k++) {
+ tmp = tmp + ma[i][k] * ma[j][k];
+ }
+ mc[i][j] = mc[i][j] * beta + tmp * alpha;
+ }
+ }
+}
+
+template <typename T> __template_static void
+syr2k(
+ clblasOrder order,
+ clblasUplo uplo,
+ clblasTranspose trans,
+ size_t N,
+ size_t K,
+ T alpha,
+ const T *A,
+ size_t lda,
+ const T *B,
+ size_t ldb,
+ T beta,
+ T *C,
+ size_t ldc)
+{
+ size_t i, j, k;
+ clblasTranspose tr =
+ trans == clblasNoTrans ? clblasNoTrans : clblasTrans;
+ MatrixAccessor<T> ma(const_cast<T*>(A), order, tr, N, K, lda);
+ MatrixAccessor<T> mb(const_cast<T*>(B), order, tr, N, K, ldb);
+ MatrixAccessor<T> mc(C, order, clblasNoTrans, N, N, ldc);
+ T tmp;
+
+ for (i = 0; i < N; i++) {
+ for (j = 0; j < N; j++) {
+ if ((uplo == clblasLower && j > i) ||
+ (uplo == clblasUpper && i > j)) {
+ continue;
+ }
+
+ tmp = ZERO<T>();
+ for (k = 0; k < K; k++) {
+ tmp = tmp + ma[i][k] * mb[j][k] + ma[j][k] * mb[i][k];
+ }
+ mc[i][j] = mc[i][j] * beta + tmp * alpha;
+ }
+ }
+}
+
+template <typename T> __template_static void
+gemv(
+ clblasOrder order,
+ clblasTranspose transA,
+ size_t M,
+ size_t N,
+ T alpha,
+ const T *A,
+ size_t lda,
+ const T *X,
+ int incx,
+ T beta,
+ T *Y,
+ int incy)
+{
+ size_t sizeX, sizeY;
+ size_t m, n;
+ T tmp;
+
+ if(transA == clblasNoTrans) {
+ sizeX = N;
+ sizeY = M;
+ }
+ else {
+ sizeX = M;
+ sizeY = N;
+ }
+
+ MatrixAccessor<T> ma(const_cast<T*>(A), order, transA, sizeY, sizeX, lda);
+ VectorAccessor<T, int> vx(const_cast<T*>(X), sizeX, incx);
+ VectorAccessor<T, int> vy(const_cast<T*>(Y), sizeY, incy);
+
+ for (m = 0; m < sizeY; m++) {
+ tmp = ZERO<T>();
+ for (n = 0; n < sizeX; n++) {
+ tmp = tmp + ma[m][n] * vx[n];
+ }
+ vy[m] = tmp * alpha + vy[m] * beta;
+ }
+}
+
+template <typename T> __template_static void
+symv(
+ clblasOrder order,
+ clblasUplo uplo,
+ size_t N,
+ T alpha,
+ const T *A,
+ size_t lda,
+ const T *X,
+ int incx,
+ T beta,
+ T *Y,
+ int incy)
+{
+ size_t m, n;
+ T tmp;
+
+ MatrixAccessor<T> ma(const_cast<T*>(A), order, clblasNoTrans, N, N, lda);
+ VectorAccessor<T, int> vx(const_cast<T*>(X), N, incx);
+ VectorAccessor<T, int> vy(const_cast<T*>(Y), N, incy);
+
+ for (m = 0; m < N; m++) {
+ tmp = ZERO<T>();
+ for (n = 0; n < N; n++) {
+ if (((uplo == clblasUpper) && (m <= n)) ||
+ ((uplo == clblasLower) && (m >= n))) {
+ tmp = tmp + ma[m][n] * vx[n];
+ }
+ else {
+ tmp = tmp + ma[n][m] * vx[n];
+ }
+ }
+ vy[m] = tmp * alpha + vy[m] * beta;
+ }
+}
diff --git a/src/library/blas/AutoGemm/AutoGemmTools/ProfileAutoGemm.cpp b/src/library/blas/AutoGemm/AutoGemmTools/ProfileAutoGemm.cpp
new file mode 100644
index 0000000..f638efb
--- /dev/null
+++ b/src/library/blas/AutoGemm/AutoGemmTools/ProfileAutoGemm.cpp
@@ -0,0 +1,1378 @@
+#include <assert.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <string>
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+//#include <Windows.h>
+#include <CL/cl.h>
+//#include "library/tools/ktest/naive/naive_blas.cpp"
+//using namespace NaiveBlas;
+#include "AutoGemmTools/AutoGemmUtil.h"
+
+#include "AutoGemmIncludes/AutoGemmKernelSelection.h"
+#include "AutoGemmIncludes/AutoGemmKernelSelectionSpecific.h"
+#include "AutoGemmIncludes/AutoGemmKernelEnumeration.h"
+
+#define SGEMM 1
+#define DGEMM 0
+#define CGEMM 0
+#define ZGEMM 0
+
+#define RANDOM_DATA 1
+#define DO_VALIDATION 0
+
+#if SGEMM
+#define DATA_TYPE float
+#define DATA_TYPE_CONSTRUCTOR(X,Y) X
+const unsigned int numTiles = sgemmNumTiles;
+const unsigned int numNonTiles = sgemmNumNonTiles;
+const unsigned int numKernels = sgemmNumKernels;
+#ifdef USER_KERNELS
+const char * const ksrFileName = "prof_user_sgemm_ksr.txt";
+const char * const rawFileName = "prof_user_sgemm_raw.csv";
+#else
+const char * const ksrFileName = "prof_sgemm_ksr.txt";
+const char * const rawFileName = "prof_sgemm_raw.csv";
+#endif
+unsigned int systemSizeMax = 8000;
+#endif
+
+#if DGEMM
+#define DATA_TYPE double
+#define DATA_TYPE_CONSTRUCTOR(X,Y) X
+const unsigned int numTiles = dgemmNumTiles;
+const unsigned int numNonTiles = dgemmNumNonTiles;
+const unsigned int numKernels = dgemmNumKernels;
+#ifdef USER_KERNELS
+const char * const ksrFileName = "prof_user_dgemm_ksr.txt";
+const char * const rawFileName = "prof_user_dgemm_raw.csv";
+#else
+const char * const ksrFileName = "prof_dgemm_ksr.txt";
+const char * const rawFileName = "prof_dgemm_raw.csv";
+#endif
+unsigned int systemSizeMax = 6000;
+#endif
+
+#if CGEMM
+#define DATA_TYPE FloatComplex
+#define DATA_TYPE_CONSTRUCTOR floatComplex
+const unsigned int numTiles = cgemmNumTiles;
+const unsigned int numNonTiles = cgemmNumNonTiles;
+const unsigned int numKernels = cgemmNumKernels;
+#ifdef USER_KERNELS
+const char * const ksrFileName = "prof_user_cgemm_ksr.txt";
+const char * const rawFileName = "prof_user_cgemm_raw.csv";
+#else
+const char * const ksrFileName = "prof_cgemm_ksr.txt";
+const char * const rawFileName = "prof_cgemm_raw.csv";
+#endif
+unsigned int systemSizeMax = 5500;
+#endif
+
+#if ZGEMM
+#define DATA_TYPE DoubleComplex
+#define DATA_TYPE_CONSTRUCTOR doubleComplex
+const unsigned int numTiles = zgemmNumTiles;
+const unsigned int numNonTiles = zgemmNumNonTiles;
+const unsigned int numKernels = zgemmNumKernels;
+#ifdef USER_KERNELS
+const char * const ksrFileName = "prof_user_zgemm_ksr.txt";
+const char * const rawFileName = "prof_user_zgemm_raw.csv";
+#else
+const char * const ksrFileName = "prof_zgemm_ksr.txt";
+const char * const rawFileName = "prof_zgemm_raw.csv";
+#endif
+unsigned int systemSizeMax = 5000;
+#endif
+
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+#define CL_CHECK(RET) \
+ if(RET != CL_SUCCESS) { \
+ printf("OpenCL error %i on line %u\n", RET, __LINE__); \
+ assert(false); \
+ }
+
+
+unsigned int **tiles;
+
+typedef struct _RuleStack {
+ unsigned int startSize;
+ unsigned int validTileIndices[numTiles];
+ unsigned int numValidTiles;
+ int fallbackTileIndex;
+ _RuleStack() : numValidTiles(0), fallbackTileIndex(-1) {}
+} RuleStack;
+
+class KernelSelectionRules {
+public:
+ RuleStack rule;
+ RuleStack history[1024];
+ unsigned int numRulesInHistory;
+ std::ostream & out;
+
+ //constructor
+ KernelSelectionRules( std::ostream & file) : numRulesInHistory(0), out(file) {
+ }
+
+ int getFastestValidTileIndex( unsigned int M, unsigned int N) {
+ for (unsigned int i = 0; i < rule.numValidTiles; i++) {
+ if ( M%tiles[rule.validTileIndices[i]][0]==0
+ && N%tiles[rule.validTileIndices[i]][1]==0) {
+ return rule.validTileIndices[i];
+ }
+ }
+ return -1;
+ }
+
+ void removeTileFromRule( unsigned int tileIdx ) {
+ int idx = -1;
+ for ( int i = 0; i < rule.numValidTiles; i++) {
+ if (rule.validTileIndices[i] == tileIdx) {
+ idx = i;
+ break;
+ }
+ }
+ if (idx >= 0) {
+ for ( int i = idx; i < rule.numValidTiles-1; i++) {
+ rule.validTileIndices[i] = rule.validTileIndices[i+1];
+ }
+ rule.numValidTiles--;
+ }
+ }
+
+ void addTileToRule( unsigned int tileIdx ) {
+ for (int i = rule.numValidTiles; i > 0; i--) {
+ rule.validTileIndices[i] = rule.validTileIndices[i-1];
+ }
+ rule.validTileIndices[0] = tileIdx;
+ rule.numValidTiles++;
+ }
+
+ bool add(
+ unsigned int M,
+ unsigned int N,
+ unsigned int *validTileIndices,
+ unsigned int numValidTiles,
+ unsigned int fallbackTileIndex ) {
+
+ printf("add(%4u,%4u) input::valid =", M, N, validTileIndices[0]);
+ for (unsigned int i = 0; i < numValidTiles; i++) {
+ printf("%ux%u, ", tiles[validTileIndices[i]][0], tiles[validTileIndices[i]][1]);
+ }
+ printf("; fallback = %ux%u\n", tiles[fallbackTileIndex][0], tiles[fallbackTileIndex][1]);
+
+
+ printf("add(%4u,%4u) rule::valid =", rule.startSize, rule.startSize, rule.validTileIndices[0]);
+ for (unsigned int i = 0; i < rule.numValidTiles; i++) {
+ printf("%ux%u, ", tiles[rule.validTileIndices[i]][0], tiles[rule.validTileIndices[i]][1]);
+ }
+ if (rule.fallbackTileIndex>=0) {
+ printf("; fallback = %ux%u", tiles[rule.fallbackTileIndex][0], tiles[rule.fallbackTileIndex][1]);
+ }
+ printf("\n");
+
+ bool mismatch = false;
+
+ // compare fallbacks
+ if (rule.fallbackTileIndex < 0) {
+ mismatch = true;
+ printf("mismatch:no fallback tile\n" );
+ rule.fallbackTileIndex = fallbackTileIndex;
+ } else {
+ if (rule.fallbackTileIndex != fallbackTileIndex) {
+ mismatch = true;
+ printf("mismatch:rule fallback was %ux%u, whereas new fallback is %u,%u\n",
+ tiles[rule.fallbackTileIndex][0], tiles[rule.fallbackTileIndex][1],
+ tiles[fallbackTileIndex][0], tiles[fallbackTileIndex][1]
+ );
+ rule.fallbackTileIndex = fallbackTileIndex;
+ }
+ }
+
+ // compare fastest valid tile
+ if (numValidTiles > 0) {
+ int ruleFastestValidTileIndex = getFastestValidTileIndex(M,N);
+ if (ruleFastestValidTileIndex < 0) {
+ // no valid tile for this M,N
+ mismatch = true;
+ printf("mismatch:no valid tile for size=%u,%u\n", M, N);
+ rule.validTileIndices[rule.numValidTiles] = validTileIndices[0];
+ rule.numValidTiles++;
+ } else {
+ if (ruleFastestValidTileIndex != validTileIndices[0]) {
+ // there is a valid tile for this M,N but it mismatches
+ mismatch = true;
+ printf("mismatch:rule tile was %ux%u, whereas fastest is %ux%u\n",
+ tiles[ruleFastestValidTileIndex][0], tiles[ruleFastestValidTileIndex][1],
+ tiles[validTileIndices[0]][0], tiles[validTileIndices[0]][1]
+ );
+ removeTileFromRule(validTileIndices[0]); // if it existed elsewhere in the rule stack
+ addTileToRule(validTileIndices[0]);
+ }
+ }
+ }
+
+ // remove retired tiles
+ for (unsigned int i = 0; i < rule.numValidTiles; i++) {
+ if ( M%tiles[rule.validTileIndices[i]][0]==0
+ && N%tiles[rule.validTileIndices[i]][1]==0) {
+
+ bool tileIsValid = false;
+ for (unsigned int j = 0; j < numValidTiles; j++) {
+ if (validTileIndices[j] == rule.validTileIndices[i]) {
+ tileIsValid = true;
+ break;
+ }
+ }
+ if (!tileIsValid) {
+ mismatch = true;
+ printf("mismatch:tile %ux%u no longer valid\n", tiles[rule.validTileIndices[i]][0], tiles[rule.validTileIndices[i]][1]);
+ removeTileFromRule( rule.validTileIndices[i] );
+ }
+ }
+ }
+
+ // if new rule, add it to history
+ if (mismatch) {
+ // update history
+ rule.startSize = sqrt(M*N)+0.5;
+ history[numRulesInHistory] = rule;
+ numRulesInHistory++;
+
+ // print what we added
+ printf("add(%4u,%4u) new::valid =", rule.startSize, rule.startSize, rule.validTileIndices[0]);
+ for (unsigned int i = 0; i < rule.numValidTiles; i++) {
+ printf("%ux%u, ", tiles[rule.validTileIndices[i]][0], tiles[rule.validTileIndices[i]][1]);
+ }
+ printf("; fallback = %ux%u\n", tiles[rule.fallbackTileIndex][0], tiles[rule.fallbackTileIndex][1]);
+
+ // write size event
+ out << " [ " << std::setw(4) << rule.startSize;
+ // write fallback
+ out << ", [ " << std::setw(2) << tiles[rule.fallbackTileIndex][0] << ", " << std::setw(2) << tiles[rule.fallbackTileIndex][1] << "]";
+ out << ", [ ";
+ if (rule.numValidTiles >= 0) {
+ out << "[ " << std::setw(2) << tiles[rule.validTileIndices[0]][0] << ", " << std::setw(2) << tiles[rule.validTileIndices[0]][1] << "]";
+ }
+ for (unsigned int i = 1; i < rule.numValidTiles; i++) {
+ out << ", [ " << std::setw(2) << tiles[rule.validTileIndices[i]][0] << ", " << std::setw(2) << tiles[rule.validTileIndices[i]][1] << "]";
+ }
+ out << " ] ], \n";
+ out.flush();
+
+ }
+
+ return mismatch;
+ }
+};
+
+
+
+
+
+
+
+
+
+template<typename T>
+void
+randomMatrix(
+ clblasOrder order,
+ size_t rows,
+ size_t columns,
+ T *A,
+ size_t lda)
+{
+ size_t r, c;
+ MatrixAccessor<T> a(A, order, clblasNoTrans, rows, columns, lda);
+
+ for (r = 0; r < rows; r++) {
+ for (c = 0; c < columns; c++) {
+#if RANDOM_DATA
+ a[r][c] = random<T>();
+#else
+ a[r][c] = DATA_TYPE_CONSTRUCTOR(1, 0);
+#endif
+ }
+ }
+}
+
+/******************************************************************************
+ * Make Gemm Kernel
+ *****************************************************************************/
+void makeGemmKernel(
+ cl_kernel *clKernel,
+ cl_command_queue clQueue,
+ const char *kernelSource,
+ const char *sourceBuildOptions,
+ const unsigned char **kernelBinary,
+ const char *binaryBuildOptions)
+{
+ cl_int err;
+ if (*clKernel) {
+ // kernel has already been built, return
+#if 0
+ // get kernel name
+ size_t kernelNameLength;
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ sizeof(kernelNameLength),
+ NULL,
+ &kernelNameLength );
+ CL_CHECK(err)
+ char *kernelName = new char[kernelNameLength];
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ kernelNameLength*sizeof(char),
+ kernelName,
+ NULL );
+ CL_CHECK(err)
+ printf("makeGemmKernel: \"%s\" already built; returning.\n", kernelName);
+ delete[] kernelName;
+#endif
+ return;
+ } else {
+ // kernel has not been built, so build it (from binary, preferably)
+ cl_context clContext;
+ cl_device_id clDevice;
+ err = clGetCommandQueueInfo( clQueue, CL_QUEUE_CONTEXT, sizeof(clContext), &clContext, NULL);
+ CL_CHECK(err)
+ err = clGetCommandQueueInfo( clQueue, CL_QUEUE_DEVICE, sizeof(clDevice), &clDevice, NULL);
+ CL_CHECK(err)
+ cl_program clProgram;
+ cl_int clBinaryStatus;
+ if (*kernelBinary) {
+ size_t kernelBinarySize = strlen((char *)*kernelBinary);
+ clProgram = clCreateProgramWithBinary(
+ clContext,
+ 1, &clDevice,
+ &kernelBinarySize, kernelBinary,
+ &clBinaryStatus, &err );
+ CL_CHECK(err)
+ err = clBuildProgram(
+ clProgram,
+ 1, &clDevice,
+ binaryBuildOptions, NULL, NULL );
+ CL_CHECK(err)
+ } else {
+ clProgram = clCreateProgramWithSource(
+ clContext,
+ 1, &kernelSource,
+ NULL, &err );
+ CL_CHECK(err)
+ err = clBuildProgram(
+ clProgram,
+ 1, &clDevice,
+ sourceBuildOptions, NULL, NULL );
+ CL_CHECK(err)
+ }
+ err = clCreateKernelsInProgram(
+ clProgram,
+ 1, clKernel,
+ NULL );
+ CL_CHECK(err)
+
+#if 0
+ // get kernel name
+ size_t kernelNameLength;
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ sizeof(kernelNameLength),
+ NULL,
+ &kernelNameLength );
+ CL_CHECK(err)
+ char *kernelName = new char[kernelNameLength];
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ kernelNameLength*sizeof(char),
+ kernelName,
+ NULL );
+ CL_CHECK(err)
+ printf("makeGemmKernel: \"%s\" built; returning.\n", kernelName);
+ delete[] kernelName;
+#endif
+ }
+}
+
+
+/****************************************************************************
+ * Compare Matrices
+ ***************************************************************************/
+template<typename T>
+bool
+compareMatrices(
+ clblasOrder order,
+ size_t rows,
+ size_t columns,
+ T *blasMatrix,
+ T *naiveMatrix,
+ size_t ld)
+{
+ size_t r, c;
+ MatrixAccessor<T> blas(blasMatrix, order, clblasNoTrans, rows, columns, ld);
+ MatrixAccessor<T> naive(naiveMatrix, order, clblasNoTrans, rows, columns, ld);
+ T blasVal, naiveVal;
+ int numPrint = 96*96;
+ bool equal = true;
+ for (r = 0; r < rows; r++) {
+ for (c = 0; c < columns; c++) {
+ blasVal = blas[r][c];
+ naiveVal = naive[r][c];
+ if (isNAN(blasVal) && isNAN(naiveVal)) {
+ continue;
+ }
+ if (blasVal != naiveVal) {
+ equal = false;
+ }
+
+ if (blasVal != naiveVal) {
+ if (numPrint-- > 0) {
+#if CGEMM || ZGEMM
+ printf("MISMATCH C[%u][%u]: gpu= %4.1f + %4.1fi, cpu= %4.1f + %4.1fi\n",
+ r, c,
+ blasVal.s[0], blasVal.s[1],
+ naiveVal.s[0], naiveVal.s[1] );
+#else
+ printf("MISMATCH C[%u][%u]: gpu= %4.1f, cpu= %4.1f\n",
+ r, c,
+ blasVal,
+ naiveVal );
+#endif
+ } else {
+ return equal;
+ }
+ }
+ }
+ }
+ return equal;
+}
+
+
+const char PLATFORM_NAME[] = "AMD Accelerated Parallel Processing";
+const char DEVICE_NAME[] = "Hawaii";
+#if SGEMM || CGEMM
+const float peakGflops = 5.24e3; // sp for W9100
+#else
+const float peakGflops = 2.62e3; // dp for W9100
+#endif
+//const float peakGflops = 696; // for R9 290 "Hawaii"
+
+
+const cl_uint offsetM = 0;
+const cl_uint offsetN = 0;
+const cl_uint offsetK = 0;
+cl_uint offA = 0;
+cl_uint offB = 0;
+cl_uint offC = 0;
+DATA_TYPE alpha = DATA_TYPE_CONSTRUCTOR(1, 0);
+cl_mem bufA = NULL;
+cl_mem bufB = NULL;
+cl_mem bufC = NULL;
+DATA_TYPE* A = NULL;
+DATA_TYPE* B = NULL;
+DATA_TYPE* C = NULL;
+DATA_TYPE* naiveC = NULL;
+const cl_uint workDim = 2;
+
+std::ofstream file;
+std::ofstream ksrFile;
+
+#if DO_VALIDATION
+const unsigned int numEnqueuesPerFlush = 1;
+const unsigned int numFlushesPerFinish = 1;
+const unsigned int numFinishes = 1;
+#else
+const unsigned int numEnqueuesPerFlush = 10;
+const unsigned int numFlushesPerFinish = 1;
+const unsigned int numFinishes = 1;
+#endif
+
+char* loadFile(const char* path);
+cl_platform_id getPlatform(const char *name);
+cl_device_id getDevice(cl_platform_id platform, const char *name);
+cl_kernel createKernel(const char *source, cl_context context,
+ const char* options, cl_int *error);
+
+
+cl_int err;
+cl_platform_id platform;
+cl_device_id device;
+cl_context_properties props[3] = { CL_CONTEXT_PLATFORM, 0, 0 };
+cl_context context;
+cl_command_queue queue;
+
+
+/****************************************************************************
+ * Benchmark Kernel
+ ***************************************************************************/
+float benchmarkKernel(
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ bool betaNonZero,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll,
+ size_t M,
+ size_t N,
+ size_t K
+ ) {
+
+
+ DATA_TYPE beta;
+ if (betaNonZero) {
+ beta = DATA_TYPE_CONSTRUCTOR(1, 0);
+ } else {
+ beta = DATA_TYPE_CONSTRUCTOR(0, 0);
+ }
+
+ bool needTileKernel = M/macroTileNumRows > 0 && N/macroTileNumCols > 0;
+ bool needRowKernel = M%macroTileNumRows > 0 && N/macroTileNumCols > 0;
+ bool needColKernel = N%macroTileNumCols > 0 && M/macroTileNumRows > 0;
+ bool needCornerKernel = M%macroTileNumRows > 0 && N%macroTileNumCols > 0;
+
+
+#if 1
+ printf("Testing: %sgemm_%s_%s%s_%s_%03u_%03u_%02u\n",
+#if SGEMM
+ "s",
+#elif DGEMM
+ "d",
+#elif CGEMM
+ "c",
+#else
+ "z",
+#endif
+ order==clblasColumnMajor ? "Col" : "Row",
+ transA==clblasTrans ? "T" : "N",
+ transB==clblasTrans ? "T" : "N",
+ betaNonZero ? "_B1" : "_B0",
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll
+ );
+#endif
+
+ //printf("M=%u, N=%u, K=%u\n", M, N, K);
+ // matrix A parameters
+ cl_uint numRowsA;
+ cl_uint numColsA;
+ if (transA == clblasTrans) {
+ numRowsA = K;
+ numColsA = M;
+ } else {
+ numRowsA = M;
+ numColsA = K;
+ }
+
+ // matrix B parameters
+ cl_uint numRowsB;
+ cl_uint numColsB;
+ if (transB == clblasTrans) {
+ numRowsB = N;
+ numColsB = K;
+ } else {
+ numRowsB = K;
+ numColsB = N;
+ }
+
+ // Matrix C
+ cl_uint numRowsC = M;
+ cl_uint numColsC = N;
+
+// leading dimension
+ cl_uint lda;
+ cl_uint ldb;
+ cl_uint ldc;
+ if (order == clblasColumnMajor) {
+ lda = numRowsA;
+ ldb = numRowsB;
+ ldc = numRowsC;
+ } else {
+ lda = numColsA;
+ ldb = numColsB;
+ ldc = numColsC;
+ }
+
+ const char *tileKernelSource;
+ const char *rowKernelSource;
+ const char *colKernelSource;
+ const char *cornerKernelSource;
+ const char *sourceBuildOptions;
+ const unsigned char *tileKernelBinary;
+ const unsigned char *rowKernelBinary;
+ const unsigned char *colKernelBinary;
+ const unsigned char *cornerKernelBinary;
+ size_t *tileKernelBinarySize = 0;
+ size_t *rowKernelBinarySize = 0;
+ size_t *colKernelBinarySize = 0;
+ size_t *cornerKernelBinarySize = 0;
+ const char *binaryBuildOptions;
+ cl_kernel *tileClKernel;
+ cl_kernel *rowClKernel;
+ cl_kernel *colClKernel;
+ cl_kernel *cornerClKernel;
+
+ unsigned int workGroupNumRows;
+ unsigned int workGroupNumCols;
+ unsigned int microTileNumRows;
+ unsigned int microTileNumCols;
+
+ //printf("Creating kernel.\n");
+ bool kernelFound =
+ gemmSelectKernelSpecific<DATA_TYPE>(
+ order,
+ transA,
+ transB,
+ betaNonZero,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ &tileKernelSource,
+ &rowKernelSource,
+ &colKernelSource,
+ &cornerKernelSource,
+ &sourceBuildOptions,
+ &tileKernelBinary,
+ &rowKernelBinary,
+ &colKernelBinary,
+ &cornerKernelBinary,
+ &tileKernelBinarySize,
+ &rowKernelBinarySize,
+ &colKernelBinarySize,
+ &cornerKernelBinarySize,
+ &binaryBuildOptions,
+ &tileClKernel,
+ &rowClKernel,
+ &colClKernel,
+ &cornerClKernel,
+ &workGroupNumRows,
+ &workGroupNumCols,
+ µTileNumRows,
+ µTileNumCols
+ );
+
+
+
+ if ( !kernelFound ) {
+ printf("ERROR: couldn't find kernel\n" );
+ }
+
+ if (needTileKernel) makeGemmKernel( tileClKernel, queue, tileKernelSource, sourceBuildOptions, &tileKernelBinary, binaryBuildOptions);
+ if (needRowKernel) makeGemmKernel( rowClKernel, queue, rowKernelSource, sourceBuildOptions, &rowKernelBinary, binaryBuildOptions);
+ if (needColKernel) makeGemmKernel( colClKernel, queue, colKernelSource, sourceBuildOptions, &colKernelBinary, binaryBuildOptions);
+ if (needCornerKernel) makeGemmKernel(cornerClKernel, queue, cornerKernelSource, sourceBuildOptions, &cornerKernelBinary, binaryBuildOptions);
+
+ /****************************************************************************
+ * Tile Kernel
+ ***************************************************************************/
+ //printf("%s", tileKernelSource);
+ unsigned int totalEnqueues = 0;
+ if (needTileKernel) {
+ err = clSetKernelArg(*tileClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ totalEnqueues++;
+ }
+ // kernel dimensions
+ const size_t localWorkSize[2] = { workGroupNumRows, workGroupNumCols };
+ size_t tileKernelGlobalWorkSize[2] = { (M/(macroTileNumRows))*workGroupNumRows, (N/(macroTileNumCols))*workGroupNumCols };
+ size_t rowKernelGlobalWorkSize[2] = { 1*workGroupNumRows, (N/(macroTileNumCols))*workGroupNumCols };
+ size_t colKernelGlobalWorkSize[2] = { (M/(macroTileNumRows))*workGroupNumRows, 1*workGroupNumCols };
+ size_t cornerKernelGlobalWorkSize[2] = { 1*workGroupNumRows, 1*workGroupNumCols };
+
+ /****************************************************************************
+ * Row Kernel (along bottom of matrix)
+ ***************************************************************************/
+ if (needRowKernel) {
+ err = clSetKernelArg(*rowClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ totalEnqueues++;
+ // kernel dimensions
+ }
+
+ /****************************************************************************
+ * Col Kernel (along side of kernel)
+ ***************************************************************************/
+ if (needColKernel) {
+ err = clSetKernelArg(*colClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ totalEnqueues++;
+ // kernel dimensions
+ }
+
+ /****************************************************************************
+ * Corner Kernel (lower left corder of kernel)
+ ***************************************************************************/
+ if (needCornerKernel) {
+ err = clSetKernelArg(*cornerClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ totalEnqueues++;
+ // kernel dimensions
+ }
+
+ totalEnqueues *= numEnqueuesPerFlush * numFlushesPerFinish * numFinishes;
+
+ cl_event kernelEvents[numEnqueuesPerFlush * numFlushesPerFinish * numFinishes * 4];
+ unsigned int kernelIdx = 0;
+ //printf("Launching %u kernels of %u x %u threads\n", totalEnqueues, globalWorkSize[0], globalWorkSize[1]);
+ for (unsigned int finishIdx = 0; finishIdx < numFinishes; finishIdx++) {
+ for (unsigned int flushIdx = 0; flushIdx < numFlushesPerFinish; flushIdx++) {
+ for (unsigned int enqIdx = 0; enqIdx < numEnqueuesPerFlush; enqIdx++) {
+ // tile kernel
+ if (needTileKernel) {
+ err = clEnqueueNDRangeKernel(queue, *tileClKernel, workDim, NULL,
+ tileKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+ }
+ // row kernel
+ if (needRowKernel) {
+ //printf("launching rowKernel %ux%u threads b/c M=%u\n", rowKernelGlobalWorkSize[0], rowKernelGlobalWorkSize[1], M);
+ err = clEnqueueNDRangeKernel(queue, *rowClKernel, workDim, NULL,
+ rowKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+ }
+ // col kernel
+ if (needColKernel) {
+ //printf("launching colKernel %ux%u threads b/c N=%u\n", colKernelGlobalWorkSize[0], colKernelGlobalWorkSize[1], N);
+ err = clEnqueueNDRangeKernel(queue, *colClKernel, workDim, NULL,
+ colKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+ }
+ // corner kernel
+ if (needCornerKernel) {
+ //printf("launching crnKernel %ux%u threads b/c M=%u, N=%u\n", cornerKernelGlobalWorkSize[0], cornerKernelGlobalWorkSize[1], M, N);
+ err = clEnqueueNDRangeKernel(queue, *cornerClKernel, workDim, NULL,
+ cornerKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+ }
+ }
+ err = clFlush(queue);
+ CL_CHECK(err);
+ }
+ err = clFinish(queue);
+ CL_CHECK(err);
+ }
+
+ cl_ulong totalNs = 0;
+ cl_ulong totalFlops = (size_t) numEnqueuesPerFlush * numFlushesPerFinish * numFinishes * (2 * M * N * K);
+#if CGEMM || ZGEMM
+ // complex
+ totalFlops *= 4;
+#endif
+
+ cl_ulong start, end;
+ for (kernelIdx = 0; kernelIdx < totalEnqueues; kernelIdx++) {
+ err = clGetEventProfilingInfo(kernelEvents[kernelIdx], CL_PROFILING_COMMAND_START,
+ sizeof(start), &start, NULL);
+ CL_CHECK(err);
+ err = clGetEventProfilingInfo(kernelEvents[kernelIdx], CL_PROFILING_COMMAND_END,
+ sizeof(end), &end, NULL);
+ CL_CHECK(err);
+ cl_ulong timeNs = end - start;
+ totalNs += timeNs;
+ }
+ double gFlops = (1.0*totalFlops) / (1.0*totalNs);
+ return gFlops;
+}
+
+
+
+/****************************************************************************
+ * Main
+ ***************************************************************************/
+int main(void) {
+ file.open(rawFileName, std::ios_base::out); // or ::app for append
+ file << "M, N, ";
+ bool printDetails = true;
+ // load tiles for precision
+ tiles = new unsigned int*[numTiles];
+ for (unsigned int i = 0; i < numTiles; i++) {
+ tiles[i] =
+#if SGEMM
+ sgemmTileEnumeration[i];
+#elif DGEMM
+ dgemmTileEnumeration[i];
+#elif CGEMM
+ cgemmTileEnumeration[i];
+#elif ZGEMM
+ zgemmTileEnumeration[i];
+#endif
+ }
+
+ for (unsigned int tileIdx = 0; tileIdx < numTiles; tileIdx++) {
+ unsigned int *tile = tiles[tileIdx];
+ file << tile[0] << "x" << tile[1] << ", ";
+ }
+ file << "<-F|T->, ";
+ for (unsigned int tileIdx = 0; tileIdx < numTiles; tileIdx++) {
+ unsigned int *tile = tiles[tileIdx];
+ file << tile[0] << "x" << tile[1] << ", ";
+ }
+ file << "fallback, fastest, would-be valid tiles\n";
+
+
+ int *fallbackBegin = new int[numTiles]; // size at which tile starts being fallback
+ int *fallbackEnd = new int[numTiles]; // size at which tile stops being fallback
+ int *validBegin = new int[numTiles]; // size at which tile starts being valid
+ int *validEnd = new int[numTiles]; // size at which tile stops being valid
+ float *fallbackScore = new float[numTiles]; // fallback score for a size
+ float *tileScore = new float[numTiles]; // tile score for a size
+ unsigned int *validTiles = new unsigned int[numTiles];
+ for (unsigned int i = 0; i < numTiles; i++) {
+ fallbackBegin[i] = -1;
+ fallbackEnd[i] = -1;
+ validBegin[i] = -1;
+ validEnd[i] = -1;
+ }
+
+ platform = getPlatform(PLATFORM_NAME);
+ assert(platform != NULL);
+ device = getDevice(platform, DEVICE_NAME);
+ assert(device != NULL);
+ props[1] = (cl_context_properties)platform;
+ context = clCreateContext(props, 1, &device, NULL, NULL, &err);
+ assert(context != NULL);
+ queue = clCreateCommandQueue(context, device, CL_QUEUE_PROFILING_ENABLE, &err);
+ assert(queue != NULL);
+
+
+ clblasOrder order = clblasColumnMajor;
+ clblasTranspose transA = clblasNoTrans;
+ clblasTranspose transB = clblasTrans;
+ bool beta = false;
+
+ unsigned int systemSizeMin = 16;
+ unsigned int systemSizeStep = 16;
+
+ //unsigned int kValues[] = {64, 512, 2048};
+ //unsigned int numKValues = 3;
+ unsigned int kValues[] = {0};
+ unsigned int numKValues = 1;
+ //unsigned int kValues[] = {4032};
+ //unsigned int numKValues = 1;
+ unsigned int kMax;
+ if (kValues[numKValues-1] > 0) {
+ kMax = kValues[numKValues-1];
+ } else {
+ kMax = systemSizeMax;
+ }
+
+
+ /******************************************************************
+ * Largest Matrix Dimension
+ *****************************************************************/
+ cl_uint numRowsA;
+ cl_uint numColsA;
+ if (transA == clblasTrans) {
+ numRowsA = kMax;
+ numColsA = systemSizeMax;
+ } else {
+ numRowsA = systemSizeMax;
+ numColsA = kMax;
+ }
+
+ // matrix B parameters
+ cl_uint numRowsB;
+ cl_uint numColsB;
+ if (transB == clblasTrans) {
+ numRowsB = systemSizeMax;
+ numColsB = kMax;
+ } else {
+ numRowsB = systemSizeMax;
+ numColsB = kMax;
+ }
+
+ // Matrix C
+ cl_uint numRowsC = systemSizeMax;
+ cl_uint numColsC = systemSizeMax;
+
+// leading dimension
+ cl_uint lda;
+ cl_uint ldb;
+ cl_uint ldc;
+ if (order == clblasColumnMajor) {
+ lda = numRowsA;
+ ldb = numRowsB;
+ ldc = numRowsC;
+ } else {
+ lda = numColsA;
+ ldb = numColsB;
+ ldc = numColsC;
+ }
+
+ /******************************************************************
+ * Allocate Matrices
+ *****************************************************************/
+ A = (DATA_TYPE*)malloc((offA + numRowsA * numColsA) * sizeof(*A));
+ assert(A != NULL);
+ randomMatrix(order, numRowsA, numColsA, A + offA, lda);
+ B = (DATA_TYPE*)malloc((offB + numRowsB * numColsB) * sizeof(*B));
+ assert(B != NULL);
+ randomMatrix(order, numRowsB, numColsB, B + offB, ldb);
+ C = (DATA_TYPE*)malloc((offC + numRowsC * numColsC) * sizeof(*C));
+ assert(C != NULL);
+ randomMatrix(order, numRowsC, numColsC, C + offC, ldc);
+ bufA = clCreateBuffer(context, CL_MEM_READ_ONLY,
+ (offA + numRowsA * numColsA) * sizeof(*A), NULL, &err);
+ CL_CHECK(err);
+ assert(bufA != NULL);
+ err = clEnqueueWriteBuffer(queue, bufA, CL_TRUE, 0,
+ (offA + numRowsA * numColsA) * sizeof(*A), A,
+ 0, NULL, NULL);
+ CL_CHECK(err);
+ assert(err == CL_SUCCESS);
+ bufB = clCreateBuffer(context, CL_MEM_READ_ONLY,
+ (offB + numRowsB * numColsB) * sizeof(*B), NULL, &err);
+ CL_CHECK(err);
+ assert(bufB != NULL);
+ err = clEnqueueWriteBuffer(queue, bufB, CL_TRUE, 0,
+ (offB + numRowsB * numColsB) * sizeof(*B), B,
+ 0, NULL, NULL);
+ CL_CHECK(err);
+ assert(err == CL_SUCCESS);
+
+ //printf("Writing to gpu buffers.\n");
+ bufC = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ (offC + numRowsC * numColsC) * sizeof(*C), NULL, &err);
+ CL_CHECK(err);
+ assert(bufC != NULL);
+ err = clEnqueueWriteBuffer(queue, bufC, CL_TRUE, 0,
+ (offC + numRowsC * numColsC) * sizeof(*C), C,
+ 0, NULL, NULL);
+ CL_CHECK(err);
+ assert(err == CL_SUCCESS);
+
+ // (1) for each system size
+ ksrFile.open( ksrFileName, std::ios_base::out); // or ::app for append
+ KernelSelectionRules ksr(ksrFile);
+ for (unsigned int systemSize = systemSizeMin; systemSize <= systemSizeMax; systemSize += systemSizeStep) {
+
+ unsigned int M = systemSize;
+ unsigned int N = systemSize;
+ file << M << ", " << N << ", ";
+
+ // reset scores for this system size
+ for (unsigned int i = 0; i < numTiles; i++) {
+ fallbackScore[i] = 0.f;
+ tileScore[i] = 0.f;
+ }
+
+ // (2) for each k size
+ for (unsigned int kIdx = 0; kIdx < numKValues; kIdx++) {
+ unsigned int K = kValues[kIdx];
+ if (K == 0) K = systemSize;
+
+ // (3) for each tile
+ for (unsigned int tileIdx = 0; tileIdx < numTiles; tileIdx++) {
+ unsigned int *tile = tiles[tileIdx];
+ unsigned int macroTileNumRows = tile[0];
+ unsigned int macroTileNumCols = tile[1];
+ unsigned int unroll = tile[2];
+ if (printDetails) printf("%4ux%4ux%4u; %ux%u; ", M, N, K, macroTileNumRows, macroTileNumCols );
+
+ /******************************************************************
+ * (4) fallback speed
+ *****************************************************************/
+ float fallbackSpeed = benchmarkKernel(
+ // non-tile
+ order, transA, transB, false,
+ // tile
+ macroTileNumRows, macroTileNumCols,
+ unroll,
+ // system
+ M-1, N-1, K
+ );
+ fallbackScore[tileIdx] += fallbackSpeed;
+
+ /******************************************************************
+ * (5) tile speed
+ *****************************************************************/
+ float tileSpeed = 0.f;
+ if (M%macroTileNumRows==0 && N%macroTileNumCols==0) {
+ tileSpeed = benchmarkKernel(
+ // non-tile
+ order, transA, transB, false,
+ // tile
+ macroTileNumRows, macroTileNumCols,
+ unroll,
+ // system
+ M, N, K
+ );
+ tileScore[tileIdx] += tileSpeed;
+ }
+
+ if (printDetails) printf("fs=%8.3f, ts=%8.3f\n", fallbackSpeed, tileSpeed );
+ } // tile sizes
+
+ } // for k
+
+ /**************************************************************
+ * (6) score is gbps averaged over k values
+ *************************************************************/
+ for (unsigned int tileIdx = 0; tileIdx < numTiles; tileIdx++) {
+ fallbackScore[tileIdx] /= numKValues;
+ file << fallbackScore[tileIdx] << ", ";
+ }
+ file << "<-F|T->, ";
+ for (unsigned int tileIdx = 0; tileIdx < numTiles; tileIdx++) {
+ tileScore[tileIdx] /= numKValues;
+ file << tileScore[tileIdx] << ", ";
+ }
+
+
+ /**************************************************************
+ * (7) get fastest fallback speed for this system size
+ *************************************************************/
+ float fastestFallbackScore = 0;
+ unsigned int fastestFallbackIdx = 0;
+ for (unsigned int tileIdx = 0; tileIdx < numTiles; tileIdx++) {
+ if (fallbackScore[tileIdx] > fastestFallbackScore) {
+ fastestFallbackScore = fallbackScore[tileIdx];
+ fastestFallbackIdx = tileIdx;
+ }
+ }
+ file << tiles[fastestFallbackIdx][0] << "x" << tiles[fastestFallbackIdx][1] << ", ";
+
+ /**************************************************************
+ * (8) ensure fallback tile has begun/ended
+ *************************************************************/
+ //if (fallbackBegin[fastestFallbackIdx] == -1) {
+ // fallbackBegin[fastestFallbackIdx] = static_cast<int>(systemSize);
+ //}
+ //fallbackEnd[fastestFallbackIdx] = static_cast<int>(systemSize); // push the end back farther
+
+
+ /**************************************************************
+ * (9) which tiles are valid for this system size
+ * - tile must be faster than fallback
+ * - there must not exist a faster tile which covers the same multiples
+ *************************************************************/
+ unsigned int numValidTiles = 0;
+ float priorFastestTileScore = 99999999;
+ for (unsigned int checkIter = 0; checkIter < numTiles; checkIter++) {
+ // find the next fastest tile
+ float fastestTileScore = -1.f;
+ int fastestTileIdx = -1;
+ for (unsigned int tileIdx = 0; tileIdx < numTiles; tileIdx++) {
+ if (tileScore[tileIdx] > fastestTileScore && (tileScore[tileIdx] < priorFastestTileScore || priorFastestTileScore < 0) ) {
+ fastestTileScore = tileScore[tileIdx];
+ fastestTileIdx = tileIdx;
+ }
+ }
+ priorFastestTileScore = fastestTileScore;
+
+ // if next fastest tile isn't faster than fallback, then quit
+ if (fastestTileScore < fastestFallbackScore-1) break;
+
+ // if the coverage of this tile is already handled by prior (faster) valid tiles, then skip it
+ bool uniqueCoverage = true;
+ for (unsigned int i = 0; i < numValidTiles; i++) {
+ if ( tiles[fastestTileIdx][0] % tiles[ validTiles[i] ][0] == 0
+ && tiles[fastestTileIdx][1] % tiles[ validTiles[i] ][1] == 0 )
+ {
+ uniqueCoverage = false;
+ break;
+ }
+ }
+ if (!uniqueCoverage) continue;
+
+ // this tile valid
+ validTiles[numValidTiles] = fastestTileIdx;
+ numValidTiles++;
+ }
+ for (unsigned int i = 0; i < numValidTiles; i++) {
+ file << tiles[validTiles[i]][0] << "x" << tiles[validTiles[i]][1] << ", ";
+ }
+
+ ksr.add(M, N, validTiles, numValidTiles, fastestFallbackIdx );
+
+ // for now, just pay attention to the fastest tile
+ //if (numValidTiles > 1) {
+ // numValidTiles = 1;
+ //}
+
+ /**************************************************************
+ * (10) ensure valid tiles have begun/ended
+ *************************************************************/
+ //for (unsigned int i = 0; i < numValidTiles; i++) {
+ // if (validBegin[ validTiles[i] ] == -1) {
+ // validBegin[ validTiles[i] ] = static_cast<int>(systemSize);
+ // }
+ // validEnd[ validTiles[i] ] = static_cast<int>(systemSize); // push the end back farther
+ //}
+
+ // print valid tiles
+ //printf("%4ux%4u; fallback = %ux%u; validTiles = ", M, N, tiles[fastestFallbackIdx][2], tiles[fastestFallbackIdx][3]);
+ //for (unsigned int i = 0; i < numValidTiles; i++) {
+ // printf("%ux%u, ", tiles[ validTiles[i] ][2], tiles[ validTiles[i] ][3]);
+ //}
+ //printf("\n");
+
+ // print tile ranges
+ //for (unsigned int i = 0; i < numTiles; i++) {
+ // printf("%4u; %ux%u fallback=[%4i, %4i] tile=[%4i, %4i]\n",
+ // systemSize, tiles[i][2], tiles[i][3],
+ // fallbackBegin[i], fallbackEnd[i],
+ // validBegin[i], validEnd[i] );
+ //}
+ //printf("\n");
+ file << "\n";
+
+
+ } // for system size
+
+ file.close();
+ ksrFile.close();
+ //err = clReleaseMemObject(bufA); CL_CHECK(err);
+ //err = clReleaseMemObject(bufB); CL_CHECK(err);
+ //err = clReleaseMemObject(bufC); CL_CHECK(err);
+ //err = clReleaseKernel(kernel); CL_CHECK(err);
+ //err = clReleaseCommandQueue(queue); CL_CHECK(err);
+ //err = clReleaseContext(context); CL_CHECK(err);
+
+ //free(A);
+ //free(B);
+ //free(C);
+ //free(naiveC);
+ //free(source);
+
+ //system("PAUSE");
+ //Sleep(5000); // ms
+ exit(EXIT_SUCCESS);
+};
+
+
+cl_platform_id
+getPlatform(const char *name)
+{
+ cl_int err;
+ cl_uint nrPlatforms, i;
+ cl_platform_id *list, platform;
+ char platformName[64];
+
+ err = clGetPlatformIDs(0, NULL, &nrPlatforms);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ return NULL;
+ }
+
+ list = (cl_platform_id*)malloc(nrPlatforms * sizeof(*list));
+ if (list == NULL) {
+ return NULL;
+ }
+
+ err = clGetPlatformIDs(nrPlatforms, list, NULL);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ free(list);
+ return NULL;
+ }
+
+ platform = NULL;
+ for (i = 0; i < nrPlatforms; i++) {
+ err = clGetPlatformInfo(list[i], CL_PLATFORM_NAME,
+ sizeof(platformName), platformName, NULL);
+ assert(err == CL_SUCCESS);
+ if ((err == CL_SUCCESS) && (strcmp(platformName, name) == 0)) {
+ platform = list[i];
+ break;
+ }
+ }
+
+ free(list);
+
+ return platform;
+}
+
+cl_device_id
+getDevice(
+ cl_platform_id platform,
+ const char *name)
+{
+
+ cl_int err;
+ cl_uint nrDevices, i;
+ cl_device_id *list, device;
+ char deviceName[64];
+
+ err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, 0, NULL, &nrDevices);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ return NULL;
+ }
+ list = (cl_device_id*)malloc(nrDevices * sizeof(*list));
+ assert(list);
+ if (list == NULL) {
+ return NULL;
+ }
+
+ err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, nrDevices, list, NULL);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ free(list);
+ return NULL;
+ }
+
+ device = NULL;
+ for (i = 0; i < nrDevices; i++) {
+ err = clGetDeviceInfo(list[i], CL_DEVICE_NAME,
+ sizeof(deviceName), deviceName, NULL);
+ assert(err == CL_SUCCESS);
+ if ((err == CL_SUCCESS) && (strcmp(deviceName, name) == 0)) {
+ device = list[i];
+ break;
+ }
+ }
+
+ free(list);
+ return device;
+}
+
+cl_kernel
+createKernel(
+ const char* source,
+ cl_context context,
+ const char* options,
+ cl_int* error)
+{
+ //printf("Kernel Source:\n%s", source );
+ cl_int err;
+ cl_device_id device;
+ cl_program program;
+ cl_kernel kernel;
+ size_t logSize;
+ char *log;
+
+ err = clGetContextInfo(context, CL_CONTEXT_DEVICES, sizeof(device), &device, NULL);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ if (error != NULL) {
+ *error = err;
+ }
+ return NULL;
+ }
+
+ program = clCreateProgramWithSource(context, 1, &source, NULL, &err);
+ assert(err == CL_SUCCESS);
+ assert(program != NULL);
+ if (program == NULL) {
+ if (error != NULL) {
+ *error = err;
+ }
+ return NULL;
+ }
+
+ err = clBuildProgram(program, 1, &device, options, NULL, NULL);
+ if (err != CL_SUCCESS) {
+ logSize = 0;
+ clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &logSize);
+ log = (char*)malloc(logSize + 1);
+ clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, logSize, log, NULL);
+ printf("=== Build Log [%lu]===\n%s\n", logSize, log);
+ free(log);
+ }
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ clReleaseProgram(program);
+ if (error != NULL) {
+ *error = err;
+ }
+ return NULL;
+ }
+
+ kernel = NULL;
+ err = clCreateKernelsInProgram(program, 1, &kernel, NULL);
+ assert(err == CL_SUCCESS);
+ assert(kernel != NULL);
+ clReleaseProgram(program);
+
+ // kernel name
+ size_t length;
+ char kernelName[64];
+ err = clGetKernelInfo(
+ kernel,
+ CL_KERNEL_FUNCTION_NAME,
+ 64,
+ kernelName,
+ &length );
+ //printf("KernelName[%lu]: %s\n", length, kernelName);
+
+ // kernel arguments
+ cl_uint numArguments;
+ err = clGetKernelInfo(
+ kernel,
+ CL_KERNEL_NUM_ARGS,
+ sizeof(numArguments),
+ &numArguments,
+ NULL );
+
+ if (error != NULL) {
+ *error = err;
+ }
+ return kernel;
+}
+
diff --git a/src/library/blas/AutoGemm/AutoGemmTools/TestAutoGemm.cpp b/src/library/blas/AutoGemm/AutoGemmTools/TestAutoGemm.cpp
new file mode 100644
index 0000000..02467fb
--- /dev/null
+++ b/src/library/blas/AutoGemm/AutoGemmTools/TestAutoGemm.cpp
@@ -0,0 +1,995 @@
+#include <assert.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <string>
+//#include <Windows.h>
+#include <CL/cl.h>
+//#include "library/tools/ktest/naive/naive_blas.cpp"
+//using namespace NaiveBlas;
+#include "AutoGemmIncludes/AutoGemmKernelSelection.h"
+#include "AutoGemmIncludes/AutoGemmKernelSelectionSpecific.h"
+#include "AutoGemmIncludes/AutoGemmKernelEnumeration.h"
+
+#include "AutoGemmUtil.h"
+
+#if 0
+// from clBLAS.h
+typedef enum clblasOrder_ {
+ clblasRowMajor,
+ clblasColumnMajor
+} clblasOrder;
+
+typedef enum clblasTranspose_ {
+ clblasNoTrans,
+ clblasTrans,
+ clblasConjTrans
+
+} clblasTranspose;
+#endif
+
+#define SGEMM 0
+#define DGEMM 1
+#define CGEMM 0
+#define ZGEMM 0
+
+#define RANDOM_DATA 1
+#define DO_VALIDATION 1
+
+#if SGEMM
+#define DATA_TYPE float
+#define DATA_TYPE_CONSTRUCTOR(X,Y) X
+const unsigned int numTiles = sgemmNumTiles;
+const unsigned int numNonTiles = sgemmNumNonTiles;
+const unsigned int numKernels = sgemmNumKernels;
+#endif
+
+#if DGEMM
+#define DATA_TYPE double
+#define DATA_TYPE_CONSTRUCTOR(X,Y) X
+const unsigned int numTiles = dgemmNumTiles;
+const unsigned int numNonTiles = dgemmNumNonTiles;
+const unsigned int numKernels = dgemmNumKernels;
+#endif
+
+#if CGEMM
+#define DATA_TYPE FloatComplex
+#define DATA_TYPE_CONSTRUCTOR floatComplex
+const unsigned int numTiles = cgemmNumTiles;
+const unsigned int numNonTiles = cgemmNumNonTiles;
+const unsigned int numKernels = cgemmNumKernels;
+#endif
+
+#if ZGEMM
+#define DATA_TYPE DoubleComplex
+#define DATA_TYPE_CONSTRUCTOR doubleComplex
+const unsigned int numTiles = zgemmNumTiles;
+const unsigned int numNonTiles = zgemmNumNonTiles;
+const unsigned int numKernels = zgemmNumKernels;
+#endif
+
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+#define CL_CHECK(RET) \
+ if(RET != CL_SUCCESS) { \
+ printf("OpenCL error %i on line %u\n", RET, __LINE__); \
+ assert(false); \
+ }
+
+
+template<typename T>
+void
+randomMatrix(
+ clblasOrder order,
+ size_t rows,
+ size_t columns,
+ T *A,
+ size_t lda)
+{
+ size_t r, c;
+ MatrixAccessor<T> a(A, order, clblasNoTrans, rows, columns, lda);
+
+ for (r = 0; r < rows; r++) {
+ for (c = 0; c < columns; c++) {
+#if RANDOM_DATA
+ a[r][c] = random<T>();
+#else
+ a[r][c] = DATA_TYPE_CONSTRUCTOR(1, 0);
+#endif
+ }
+ }
+}
+
+template<typename T>
+bool
+compareMatrices(
+ clblasOrder order,
+ size_t rows,
+ size_t columns,
+ T *blasMatrix,
+ T *naiveMatrix,
+ size_t ld)
+{
+ size_t r, c;
+ MatrixAccessor<T> blas(blasMatrix, order, clblasNoTrans, rows, columns, ld);
+ MatrixAccessor<T> naive(naiveMatrix, order, clblasNoTrans, rows, columns, ld);
+ T blasVal, naiveVal;
+ int numPrint = 96*96;
+ bool equal = true;
+ for (r = 0; r < rows; r++) {
+ for (c = 0; c < columns; c++) {
+ blasVal = blas[r][c];
+ naiveVal = naive[r][c];
+ if (isNAN(blasVal) && isNAN(naiveVal)) {
+ continue;
+ }
+ if (blasVal != naiveVal) {
+ equal = false;
+ }
+
+ if (blasVal != naiveVal) {
+ if (numPrint-- > 0) {
+#if CGEMM || ZGEMM
+ printf("MISMATCH C[%u][%u]: gpu= %4.1f + %4.1fi, cpu= %4.1f + %4.1fi\n",
+ r, c,
+ blasVal.s[0], blasVal.s[1],
+ naiveVal.s[0], naiveVal.s[1] );
+#else
+ printf("MISMATCH C[%u][%u]: gpu= %4.1f, cpu= %4.1f\n",
+ r, c,
+ blasVal,
+ naiveVal );
+#endif
+ } else {
+ return equal;
+ }
+ }
+ }
+ }
+ return equal;
+}
+
+
+const char PLATFORM_NAME[] = "AMD Accelerated Parallel Processing";
+const char DEVICE_NAME[] = "Hawaii";
+#if SGEMM || CGEMM
+const float peakGflops = 5.24e3; // sp for W9100
+#else
+const float peakGflops = 2.62e3; // dp for W9100
+#endif
+//const float peakGflops = 696; // for R9 290 "Hawaii"
+
+
+const cl_uint offsetM = 0;
+const cl_uint offsetN = 0;
+const cl_uint offsetK = 0;
+cl_uint offA = 0;
+cl_uint offB = 0;
+cl_uint offC = 0;
+DATA_TYPE alpha = DATA_TYPE_CONSTRUCTOR(1, 0);
+cl_mem bufA = NULL;
+cl_mem bufB = NULL;
+cl_mem bufC = NULL;
+DATA_TYPE* A = NULL;
+DATA_TYPE* B = NULL;
+DATA_TYPE* C = NULL;
+DATA_TYPE* naiveC = NULL;
+
+const cl_uint workDim = 2;
+
+#if DO_VALIDATION
+const unsigned int numEnqueuesPerFlush = 1;
+const unsigned int numFlushesPerFinish = 1;
+const unsigned int numFinishes = 1;
+#else
+const unsigned int numEnqueuesPerFlush = 2;
+const unsigned int numFlushesPerFinish = 2;
+const unsigned int numFinishes = 2;
+#endif
+
+cl_platform_id getPlatform(const char *name);
+cl_device_id getDevice(cl_platform_id platform, const char *name);
+cl_kernel createKernel(const char *source, cl_context context,
+ const char* options, cl_int *error);
+
+
+
+void testKernelParameterCombination(
+ unsigned int columnMajorInt,
+ unsigned int transAInt,
+ unsigned int transBInt,
+ unsigned int betaNonZero,
+ unsigned int macroTileNumRows,
+ unsigned int macroTileNumCols,
+ unsigned int unroll,
+ unsigned int mSpill,
+ unsigned int nSpill ) {
+
+
+ DATA_TYPE beta;
+ if (betaNonZero) {
+ beta = DATA_TYPE_CONSTRUCTOR(1, 0);
+ } else {
+ beta = DATA_TYPE_CONSTRUCTOR(0, 0);
+ }
+
+
+ // how large of a matrix to test?
+#if DO_VALIDATION
+ size_t M = 16*macroTileNumRows;
+ size_t N = 16*macroTileNumCols;
+ size_t K = 16*unroll;
+#else
+ //if (mSpill || nSpill || unroll==1 || transAInt==1 || transBInt==0) return;
+
+ if (mSpill || nSpill || unroll==1 ) return;
+
+ size_t M = 22*macroTileNumRows;
+ size_t N = 24*macroTileNumCols;
+ size_t K = 2*64*90+unroll;
+#endif
+ if (mSpill) {
+ M += 1;
+ }
+ if (nSpill) {
+ N += 1;
+ }
+
+
+#if 1
+ printf("Testing: %sgemm_%s%s_B%u_MX%03u_NX%03u_KX%02u\n",
+#if SGEMM
+ "s",
+#elif DGEMM
+ "d",
+#elif CGEMM
+ "c",
+#else
+ "z",
+#endif
+ transAInt ? "T" : "N",
+ transBInt ? "T" : "N",
+ betaNonZero ? 1 : 0,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll );
+#endif
+
+ //printf("M=%u, N=%u, K=%u\n", M, N, K);
+ // matrix A parameters
+ clblasTranspose transA;
+ cl_uint numRowsA;
+ cl_uint numColsA;
+ if (transAInt) {
+ transA = clblasTrans;
+ numRowsA = K;
+ numColsA = M;
+ } else {
+ transA = clblasNoTrans;
+ numRowsA = M;
+ numColsA = K;
+ }
+
+ // matrix B parameters
+ clblasTranspose transB;
+ cl_uint numRowsB;
+ cl_uint numColsB;
+ if (transBInt) {
+ transB = clblasTrans;
+ numRowsB = N;
+ numColsB = K;
+ } else {
+ transB = clblasNoTrans;
+ numRowsB = K;
+ numColsB = N;
+ }
+
+ // Matrix C
+ cl_uint numRowsC = M;
+ cl_uint numColsC = N;
+
+// leading dimension
+ clblasOrder order;
+ cl_uint lda;
+ cl_uint ldb;
+ cl_uint ldc;
+ if (columnMajorInt) {
+ order = clblasColumnMajor;
+ lda = numRowsA;
+ ldb = numRowsB;
+ ldc = numRowsC;
+ } else {
+ order = clblasRowMajor;
+ lda = numColsA;
+ ldb = numColsB;
+ ldc = numColsC;
+ }
+
+ cl_int err;
+ cl_platform_id platform;
+ cl_device_id device;
+ cl_context_properties props[3] = { CL_CONTEXT_PLATFORM, 0, 0 };
+ cl_context context;
+ cl_command_queue queue;
+
+ platform = getPlatform(PLATFORM_NAME);
+ assert(platform != NULL);
+ device = getDevice(platform, DEVICE_NAME);
+ assert(device != NULL);
+ props[1] = (cl_context_properties)platform;
+ context = clCreateContext(props, 1, &device, NULL, NULL, &err);
+ assert(context != NULL);
+ queue = clCreateCommandQueue(context, device, CL_QUEUE_PROFILING_ENABLE, &err);
+ assert(queue != NULL);
+
+ //printf("Allocating matrices.\n");
+ A = (DATA_TYPE*)malloc((offA + numRowsA * numColsA) * sizeof(*A));
+ assert(A != NULL);
+ randomMatrix(order, numRowsA, numColsA, A + offA, lda);
+ B = (DATA_TYPE*)malloc((offB + numRowsB * numColsB) * sizeof(*B));
+ assert(B != NULL);
+ randomMatrix(order, numRowsB, numColsB, B + offB, ldb);
+ C = (DATA_TYPE*)malloc((offC + numRowsC * numColsC) * sizeof(*C));
+ assert(C != NULL);
+ randomMatrix(order, numRowsC, numColsC, C + offC, ldc);
+ naiveC = (DATA_TYPE*)malloc((offC + numRowsC * numColsC) * sizeof(*naiveC));
+ assert(naiveC != NULL);
+ memcpy(naiveC, C, (offC + numRowsC * numColsC) * sizeof(*C));
+
+#if DO_VALIDATION
+ //printf("Running naive gemm.\n");
+ gemm(order, transA, transB, M, N, K, alpha, A + offA, lda, B + offB, ldb, beta, naiveC + offC, ldc);
+#endif
+ bufA = clCreateBuffer(context, CL_MEM_READ_ONLY,
+ (offA + numRowsA * numColsA) * sizeof(*A), NULL, &err);
+ CL_CHECK(err);
+ assert(bufA != NULL);
+ err = clEnqueueWriteBuffer(queue, bufA, CL_TRUE, 0,
+ (offA + numRowsA * numColsA) * sizeof(*A), A,
+ 0, NULL, NULL);
+ CL_CHECK(err);
+ assert(err == CL_SUCCESS);
+ bufB = clCreateBuffer(context, CL_MEM_READ_ONLY,
+ (offB + numRowsB * numColsB) * sizeof(*B), NULL, &err);
+ CL_CHECK(err);
+ assert(bufB != NULL);
+ err = clEnqueueWriteBuffer(queue, bufB, CL_TRUE, 0,
+ (offB + numRowsB * numColsB) * sizeof(*B), B,
+ 0, NULL, NULL);
+ CL_CHECK(err);
+ assert(err == CL_SUCCESS);
+
+ //printf("Writing to gpu buffers.\n");
+ bufC = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ (offC + numRowsC * numColsC) * sizeof(*C), NULL, &err);
+ CL_CHECK(err);
+ assert(bufC != NULL);
+ err = clEnqueueWriteBuffer(queue, bufC, CL_TRUE, 0,
+ (offC + numRowsC * numColsC) * sizeof(*C), C,
+ 0, NULL, NULL);
+ CL_CHECK(err);
+ assert(err == CL_SUCCESS);
+
+ float optimalNumElementsPerWorkItem = 1;
+
+ const char *tileKernelSource;
+ const char *rowKernelSource;
+ const char *colKernelSource;
+ const char *cornerKernelSource;
+ const char *sourceBuildOptions;
+ const unsigned char *tileKernelBinary;
+ const unsigned char *rowKernelBinary;
+ const unsigned char *colKernelBinary;
+ const unsigned char *cornerKernelBinary;
+ size_t *tileKernelBinarySize = 0;
+ size_t *rowKernelBinarySize = 0;
+ size_t *colKernelBinarySize = 0;
+ size_t *cornerKernelBinarySize = 0;
+ const char *binaryBuildOptions;
+ cl_kernel *tileClKernel;
+ cl_kernel *rowClKernel;
+ cl_kernel *colClKernel;
+ cl_kernel *cornerClKernel;
+ unsigned int workGroupNumRows;
+ unsigned int workGroupNumCols;
+ unsigned int microTileNumRows;
+ unsigned int microTileNumCols;
+ unsigned int retUnroll;
+
+#if 0
+ //printf("Creating kernel.\n");
+ gemmSelectKernel<DATA_TYPE>(
+ order,
+ transA,
+ transB,
+ M,
+ N,
+ K,
+ betaNonZero==1,
+ optimalNumElementsPerWorkItem,
+ &tileKernelSource,
+ &rowKernelSource,
+ &colKernelSource,
+ &cornerKernelSource,
+ &sourceBuildOptions,
+ &tileKernelBinary,
+ &rowKernelBinary,
+ &colKernelBinary,
+ &cornerKernelBinary,
+ &tileKernelBinarySize,
+ &rowKernelBinarySize,
+ &colKernelBinarySize,
+ &cornerKernelBinarySize,
+ &binaryBuildOptions,
+ &tileClKernel,
+ &rowClKernel,
+ &colClKernel,
+ &cornerClKernel,
+ &workGroupNumRows,
+ &workGroupNumCols,
+ µTileNumRows,
+ µTileNumCols,
+ &retUnroll
+ );
+ bool kernelFound = tileKernelSource != NULL;
+#else
+ bool kernelFound = gemmSelectKernelSpecific<DATA_TYPE>(
+ order,
+ transA,
+ transB,
+ betaNonZero==1,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ &tileKernelSource,
+ &rowKernelSource,
+ &colKernelSource,
+ &cornerKernelSource,
+ &sourceBuildOptions,
+ &tileKernelBinary,
+ &rowKernelBinary,
+ &colKernelBinary,
+ &cornerKernelBinary,
+ &tileKernelBinarySize,
+ &rowKernelBinarySize,
+ &colKernelBinarySize,
+ &cornerKernelBinarySize,
+ &binaryBuildOptions,
+ &tileClKernel,
+ &rowClKernel,
+ &colClKernel,
+ &cornerClKernel,
+ &workGroupNumRows,
+ &workGroupNumCols,
+ µTileNumRows,
+ µTileNumCols
+ );
+#endif
+
+ if ( !kernelFound ) {
+ printf("ERROR: selected kernel doesn't match desired kernel: %u, %u, %u, %u, %u\n",
+ workGroupNumRows,
+ workGroupNumCols,
+ microTileNumRows,
+ microTileNumCols,
+ unroll
+ );
+ }
+
+ /****************************************************************************
+ * Tile Kernel
+ ***************************************************************************/
+ //printf("%s", tileKernelSource);
+ assert(tileKernelSource != NULL);
+ *tileClKernel = createKernel(tileKernelSource, context, sourceBuildOptions, &err);
+ assert(tileClKernel != NULL);
+ err = clSetKernelArg(*tileClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ // kernel dimensions
+ const size_t localWorkSize[2] = { workGroupNumRows, workGroupNumCols };
+ size_t tileKernelGlobalWorkSize[2] = { (M/(macroTileNumRows))*workGroupNumRows, (N/(macroTileNumCols))*workGroupNumCols };
+ size_t rowKernelGlobalWorkSize[2] = { 1*workGroupNumRows, (N/(macroTileNumCols))*workGroupNumCols };
+ size_t colKernelGlobalWorkSize[2] = { (M/(macroTileNumRows))*workGroupNumRows, 1*workGroupNumCols };
+ size_t cornerKernelGlobalWorkSize[2] = { 1*workGroupNumRows, 1*workGroupNumCols };
+
+ /****************************************************************************
+ * Row Kernel (along bottom of matrix)
+ ***************************************************************************/
+ if (mSpill) {
+ assert(rowKernelSource != NULL);
+ *rowClKernel = createKernel(rowKernelSource, context, sourceBuildOptions, &err);
+ assert(rowClKernel != NULL);
+ err = clSetKernelArg(*rowClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*rowClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ // kernel dimensions
+ }
+
+ /****************************************************************************
+ * Col Kernel (along side of kernel)
+ ***************************************************************************/
+ if (nSpill) {
+ assert(colKernelSource != NULL);
+ *colClKernel = createKernel(colKernelSource, context, sourceBuildOptions, &err);
+ assert(colClKernel != NULL);
+ err = clSetKernelArg(*colClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*colClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ // kernel dimensions
+ }
+
+ /****************************************************************************
+ * Corner Kernel (lower left corder of kernel)
+ ***************************************************************************/
+ if (mSpill && nSpill) {
+ assert(cornerKernelSource != NULL);
+ *cornerClKernel = createKernel(cornerKernelSource, context, sourceBuildOptions, &err);
+ assert(cornerClKernel != NULL);
+ err = clSetKernelArg(*cornerClKernel, 0, sizeof(cl_mem), &bufA); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 1, sizeof(cl_mem), &bufB); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 2, sizeof(cl_mem), &bufC); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 3, sizeof(DATA_TYPE), &alpha); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 4, sizeof(DATA_TYPE), &beta); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 5, sizeof(cl_uint), &M); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 6, sizeof(cl_uint), &N); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 7, sizeof(cl_uint), &K); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 8, sizeof(cl_uint), &lda); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 9, sizeof(cl_uint), &ldb); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 10, sizeof(cl_uint), &ldc); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 11, sizeof(cl_uint), &offA); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 12, sizeof(cl_uint), &offB); CL_CHECK(err);
+ err = clSetKernelArg(*cornerClKernel, 13, sizeof(cl_uint), &offC); CL_CHECK(err);
+ // kernel dimensions
+ }
+
+ unsigned int totalEnqueues = numEnqueuesPerFlush * numFlushesPerFinish * numFinishes;
+ if (mSpill || nSpill) {
+ totalEnqueues *= 2;
+ }
+ if (mSpill && nSpill) {
+ totalEnqueues *= 2;
+ }
+ cl_event kernelEvents[numEnqueuesPerFlush * numFlushesPerFinish * numFinishes * 4];
+ unsigned int kernelIdx = 0;
+ //printf("Launching %u kernels of %u x %u threads\n", totalEnqueues, globalWorkSize[0], globalWorkSize[1]);
+ for (unsigned int finishIdx = 0; finishIdx < numFinishes; finishIdx++) {
+ for (unsigned int flushIdx = 0; flushIdx < numFlushesPerFinish; flushIdx++) {
+ for (unsigned int enqIdx = 0; enqIdx < numEnqueuesPerFlush; enqIdx++) {
+ // tile kernel
+ err = clEnqueueNDRangeKernel(queue, *tileClKernel, workDim, NULL,
+ tileKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+#if 1
+ // row kernel
+ if (mSpill) {
+ printf("launching rowKernel %ux%u threads b/c M=%u\n", rowKernelGlobalWorkSize[0], rowKernelGlobalWorkSize[1], M);
+ err = clEnqueueNDRangeKernel(queue, *rowClKernel, workDim, NULL,
+ rowKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+ }
+ // col kernel
+ if (nSpill) {
+ printf("launching colKernel %ux%u threads b/c N=%u\n", colKernelGlobalWorkSize[0], colKernelGlobalWorkSize[1], N);
+ err = clEnqueueNDRangeKernel(queue, *colClKernel, workDim, NULL,
+ colKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+ }
+ // corner kernel
+ if (mSpill && nSpill) {
+ printf("launching crnKernel %ux%u threads b/c M=%u, N=%u\n", cornerKernelGlobalWorkSize[0], cornerKernelGlobalWorkSize[1], M, N);
+ err = clEnqueueNDRangeKernel(queue, *cornerClKernel, workDim, NULL,
+ cornerKernelGlobalWorkSize, localWorkSize, 0, NULL, &kernelEvents[kernelIdx]);
+ CL_CHECK(err);
+ kernelIdx++;
+ }
+#endif
+ }
+ err = clFlush(queue);
+ CL_CHECK(err);
+ }
+ err = clFinish(queue);
+ CL_CHECK(err);
+ }
+
+#if DO_VALIDATION
+#else
+ cl_ulong start, end;
+ for (kernelIdx = 0; kernelIdx < totalEnqueues; kernelIdx++) {
+ err = clGetEventProfilingInfo(kernelEvents[kernelIdx], CL_PROFILING_COMMAND_START,
+ sizeof(start), &start, NULL);
+ CL_CHECK(err);
+ err = clGetEventProfilingInfo(kernelEvents[kernelIdx], CL_PROFILING_COMMAND_END,
+ sizeof(end), &end, NULL);
+ CL_CHECK(err);
+ cl_ulong timeNs = end - start;
+ cl_ulong totalFlops;
+ if (!mSpill && !nSpill) {
+ totalFlops = 2*((cl_ulong)M)*N*K;
+ } else if (mSpill && !nSpill) {
+ if (kernelIdx%2==0) {
+ totalFlops = 2*((cl_ulong)M)*N*K;
+ } else {
+ totalFlops = 2*((cl_ulong)macroTileNumRows)*N*K;
+ }
+ } else if (nSpill && !mSpill) {
+ if (kernelIdx%2==0) {
+ totalFlops = 2*((cl_ulong)M)*N*K;
+ } else {
+ totalFlops = 2*((cl_ulong)M)*macroTileNumCols*K;
+ }
+ } else {
+ if (kernelIdx%4==0) {
+ totalFlops = 2*((cl_ulong)M)*N*K;
+ } else if (kernelIdx%4==1) {
+ totalFlops = 2*((cl_ulong)macroTileNumRows)*N*K;
+ } else if (kernelIdx%4==2) {
+ totalFlops = 2*((cl_ulong)M)*macroTileNumCols*K;
+ } else {
+ totalFlops = 2*((cl_ulong)macroTileNumRows)*macroTileNumCols*K;
+ }
+ }
+#if CGEMM || ZGEMM
+ // complex
+ totalFlops *= 4;
+#endif
+ double gFlops = (1.0*totalFlops) / (1.0*timeNs);
+ printf("%12llu flops in %12llu ns = %7.1f Gflop/s (%5.1f%% of peak)\n", totalFlops, timeNs, gFlops, 100*gFlops/peakGflops);
+ }
+#endif
+
+ err = clEnqueueReadBuffer(queue, bufC, CL_TRUE, 0,
+ (offC + numRowsC * numColsC) * sizeof(*C), C,
+ 0, NULL, NULL);
+ CL_CHECK(err);
+
+#if DO_VALIDATION
+ bool equal = compareMatrices(order, numRowsC, numColsC, C + offC, naiveC + offC, ldc);
+
+ printf("%s_%s%s_%03u_%03u_%u_%02ux%02u_%ux%u%s%s%s%s",
+#if SGEMM
+ "sgemm",
+#endif
+#if DGEMM
+ "dgemm",
+#endif
+#if CGEMM
+ "cgemm",
+#endif
+#if ZGEMM
+ "zgemm",
+#endif
+ transAInt ? "T" : "N",
+ transBInt ? "T" : "N",
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ workGroupNumRows,
+ workGroupNumCols,
+ microTileNumRows,
+ microTileNumCols,
+ columnMajorInt ? "_ColumnMajor" : "_RowMajor",
+ mSpill ? "_1" : "_0",
+ nSpill ? "_1" : "_0",
+ betaNonZero ? "_BETA" : "" );
+
+ if (equal) {
+ printf(" - passed\n\n");
+ }
+ else {
+ printf(" - failed\n\n");
+ printf("%s", tileKernelSource );
+ }
+ fflush(stdout);
+ system("PAUSE");
+#endif
+
+ err = clReleaseMemObject(bufA); CL_CHECK(err);
+ err = clReleaseMemObject(bufB); CL_CHECK(err);
+ err = clReleaseMemObject(bufC); CL_CHECK(err);
+ //err = clReleaseKernel(kernel); CL_CHECK(err);
+ err = clReleaseCommandQueue(queue); CL_CHECK(err);
+ err = clReleaseContext(context); CL_CHECK(err);
+
+ free(A);
+ free(B);
+ free(C);
+ free(naiveC);
+ //free(source);
+}
+
+
+
+int main(void) {
+
+#if 0
+ srand((unsigned int)time(NULL));
+
+ unsigned int **kernels = new unsigned int*[numKernels];
+ for (unsigned int i = 0; i < numKernels; i++) {
+ kernels[i] =
+#if SGEMM
+ sgemmKernelEnumeration[i];
+#elif DGEMM
+ dgemmKernelEnumeration[i];
+#elif CGEMM
+ cgemmKernelEnumeration[i];
+#elif ZGEMM
+ zgemmKernelEnumeration[i];
+#endif
+ }
+
+
+ for (unsigned int kernelIdx = 0; kernelIdx < numKernels; kernelIdx++) {
+ printf("kernelIdx = %u\n", kernelIdx);
+ /* {isColumnMajor, transA, transB, betaNonZero, wgNumRows, wgNumCols, mtNumRows, mtNumCols, }*/
+ unsigned int *kernelParameters = kernels[kernelIdx];
+
+ unsigned int columnMajor = kernelParameters[0];
+ unsigned int transA = kernelParameters[1];
+ unsigned int transB = kernelParameters[2];
+ unsigned int betaNonZero = kernelParameters[3];
+ unsigned int macroTileNumRows = kernelParameters[4];
+ unsigned int macroTileNumCols = kernelParameters[5];
+ unsigned int unroll = kernelParameters[6];
+ unsigned int mSpill = kernelParameters[7];
+ unsigned int nSpill = kernelParameters[8];
+
+ testKernelParameterCombination(
+ columnMajor,
+ transA,
+ transB,
+ betaNonZero,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ mSpill,
+ nSpill );
+ } // end for
+#else
+
+ unsigned int columnMajor = 1;
+ unsigned int transA = 0;
+ unsigned int transB = 1;
+ unsigned int beta = 0;
+ unsigned int macroTileNumRows = 16*4;
+ unsigned int macroTileNumCols = 16*4;
+ unsigned int unroll = 8;
+ unsigned int mSpill = 0;
+ unsigned int nSpill = 0;
+
+
+ testKernelParameterCombination(
+ columnMajor,
+ transA,
+ transB,
+ true,
+ macroTileNumRows,
+ macroTileNumCols,
+ unroll,
+ mSpill,
+ nSpill );
+
+#endif
+
+
+ //system("PAUSE");
+ //Sleep(5000); // ms
+ exit(EXIT_SUCCESS);
+};
+
+
+cl_platform_id
+getPlatform(const char *name)
+{
+ cl_int err;
+ cl_uint nrPlatforms, i;
+ cl_platform_id *list, platform;
+ char platformName[64];
+
+ err = clGetPlatformIDs(0, NULL, &nrPlatforms);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ return NULL;
+ }
+
+ list = (cl_platform_id*)malloc(nrPlatforms * sizeof(*list));
+ if (list == NULL) {
+ return NULL;
+ }
+
+ err = clGetPlatformIDs(nrPlatforms, list, NULL);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ free(list);
+ return NULL;
+ }
+
+ platform = NULL;
+ for (i = 0; i < nrPlatforms; i++) {
+ err = clGetPlatformInfo(list[i], CL_PLATFORM_NAME,
+ sizeof(platformName), platformName, NULL);
+ assert(err == CL_SUCCESS);
+ if ((err == CL_SUCCESS) && (strcmp(platformName, name) == 0)) {
+ platform = list[i];
+ break;
+ }
+ }
+
+ free(list);
+ return platform;
+}
+
+cl_device_id
+getDevice(
+ cl_platform_id platform,
+ const char *name)
+{
+
+ cl_int err;
+ cl_uint nrDevices, i;
+ cl_device_id *list, device;
+ char deviceName[64];
+
+ err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, 0, NULL, &nrDevices);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ return NULL;
+ }
+ list = (cl_device_id*)malloc(nrDevices * sizeof(*list));
+ assert(list);
+ if (list == NULL) {
+ return NULL;
+ }
+
+ err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, nrDevices, list, NULL);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ free(list);
+ return NULL;
+ }
+
+ device = NULL;
+ for (i = 0; i < nrDevices; i++) {
+ err = clGetDeviceInfo(list[i], CL_DEVICE_NAME,
+ sizeof(deviceName), deviceName, NULL);
+ assert(err == CL_SUCCESS);
+ if ((err == CL_SUCCESS) && (strcmp(deviceName, name) == 0)) {
+ device = list[i];
+ break;
+ }
+ }
+
+ free(list);
+ return device;
+}
+
+cl_kernel
+createKernel(
+ const char* source,
+ cl_context context,
+ const char* options,
+ cl_int* error)
+{
+
+ printf("BuildOptions: %s\n", options );
+
+ //printf("Kernel Source:\n%s", source );
+ cl_int err;
+ cl_device_id device;
+ cl_program program;
+ cl_kernel kernel;
+ size_t logSize;
+ char *log;
+
+ err = clGetContextInfo(context, CL_CONTEXT_DEVICES, sizeof(device), &device, NULL);
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ if (error != NULL) {
+ *error = err;
+ }
+ return NULL;
+ }
+
+ program = clCreateProgramWithSource(context, 1, &source, NULL, &err);
+ assert(err == CL_SUCCESS);
+ assert(program != NULL);
+ if (program == NULL) {
+ if (error != NULL) {
+ *error = err;
+ }
+ return NULL;
+ }
+
+ err = clBuildProgram(program, 1, &device, options, NULL, NULL);
+ if (err != CL_SUCCESS) {
+ logSize = 0;
+ clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &logSize);
+ log = (char*)malloc(logSize + 1);
+ clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, logSize, log, NULL);
+ printf("=== Build Log [%lu]===\n%s\n", logSize, log);
+ free(log);
+ }
+ assert(err == CL_SUCCESS);
+ if (err != CL_SUCCESS) {
+ clReleaseProgram(program);
+ if (error != NULL) {
+ *error = err;
+ }
+ return NULL;
+ }
+
+ kernel = NULL;
+ cl_uint num_kernels_ret;
+ err = clCreateKernelsInProgram(program, 0, NULL, &num_kernels_ret);
+ err = clCreateKernelsInProgram(program, 1, &kernel, NULL);
+ assert(err == CL_SUCCESS);
+ assert(kernel != NULL);
+ clReleaseProgram(program);
+
+ // kernel name
+ size_t length;
+ char kernelName[64];
+ err = clGetKernelInfo(
+ kernel,
+ CL_KERNEL_FUNCTION_NAME,
+ 64,
+ kernelName,
+ &length );
+ printf("KernelName[%lu]: %s\n", length, kernelName);
+
+ // kernel arguments
+ cl_uint numArguments;
+ err = clGetKernelInfo(
+ kernel,
+ CL_KERNEL_NUM_ARGS,
+ sizeof(numArguments),
+ &numArguments,
+ NULL );
+
+ if (error != NULL) {
+ *error = err;
+ }
+ return kernel;
+}
+
diff --git a/src/library/blas/AutoGemm/Common.py b/src/library/blas/AutoGemm/Common.py
new file mode 100644
index 0000000..526d947
--- /dev/null
+++ b/src/library/blas/AutoGemm/Common.py
@@ -0,0 +1,60 @@
+################################################################################
+# Auto-Gemm
+################################################################################
+
+outputPath = ""
+clCompilerVersion = "2.0"
+
+def setClCompilerVersion(version):
+ global clCompilerVersion
+ clCompilerVersion = version
+
+def getClCompilerVersion():
+ global clCompilerVersion
+ return clCompilerVersion
+
+def setOutputPath(path):
+ global outputPath
+ outputPath = path + "/"
+
+def getOutputPath():
+ global outputPath
+ return outputPath
+
+def getRelativeKernelSourcePath():
+ return "AutoGemmKernelSources/"
+
+def getRelativeKernelBinaryPath():
+ return "AutoGemmKernelBinaries/"
+
+def getRelativeIncludePath():
+ return "AutoGemmIncludes/"
+
+def getKernelSourcePath():
+ return getOutputPath() + getRelativeKernelSourcePath()
+
+def getKernelBinaryPath():
+ return getOutputPath() + getRelativeKernelBinaryPath()
+
+def getIncludePath():
+ return getOutputPath() + getRelativeIncludePath()
+
+def getAutoGemmHeader():
+ return (
+ "/*******************************************************************************\n"
+ " * This file was auto-generated using the AutoGemm.py python script.\n"
+ " * DO NOT modify this file! Instead, make changes to scripts in\n"
+ " * clBLAS/src/library/blas/AutoGemm/ then re-generate files\n"
+ " * (otherwise local changes will be lost after re-generation).\n"
+ " ******************************************************************************/\n\n"
+ )
+
+hostDataChar = { "s":"s", "d":"d", "c":"c", "z":"z" }
+hostDataType = { "s":"float", "d":"double", "c":"float2", "z":"double2" }
+openclDataType = { "s":"float", "d":"double", "c":"float2", "z":"double2" }
+
+precisionInt = { "s":0, "d":1, "c":2, "z":3 }
+orderInt = { "clblasRowMajor":0, "clblasColumnMajor":1 }
+transposeInt = { "N":0, "T":1, "C":2 }
+
+
diff --git a/src/library/blas/AutoGemm/Includes.py b/src/library/blas/AutoGemm/Includes.py
new file mode 100644
index 0000000..d656592
--- /dev/null
+++ b/src/library/blas/AutoGemm/Includes.py
@@ -0,0 +1,458 @@
+import os
+import sys
+import getopt
+import Common
+import AutoGemmParameters
+import KernelParameters
+
+################################################################################
+# SINC - Kernel Source Includes
+################################################################################
+class KernelSourceIncludes:
+
+ ##############################################################################
+ # SINC - default constructor
+ ##############################################################################
+ def __init__(self):
+ self.incFileName = Common.getIncludePath() + "AutoGemmKernelSources.h"
+ self.incFile = open(self.incFileName, "w")
+ self.incFile.write( Common.getAutoGemmHeader() )
+ self.incStr = "#ifndef AUTOGEMM_KERNEL_SOURCE_INCLUDES_H\n"
+ self.incStr += "#define AUTOGEMM_KERNEL_SOURCE_INCLUDES_H\n"
+ self.incStr += "\n"
+
+ self.cppFileName = Common.getIncludePath() + "AutoGemmKernelSources.cpp"
+ self.cppFile = open(self.cppFileName, "w")
+ self.cppFile.write( Common.getAutoGemmHeader() )
+ self.cppStr = "\n"
+ self.cppStr += "#include \"%sAutoGemmKernelSources.h\"\n" % Common.getRelativeIncludePath()
+ self.cppStr += "#include \"UserGemmKernelSources/UserGemmKernelSourceIncludes.cpp\"\n"
+ #self.cppStr += "#include \"UserGemmKernelSources/UserGemmKernelSources.cpp\"\n"
+
+ def addKernel(self, kernel):
+ kernelName = kernel.getName()
+ self.incStr += "extern const unsigned int %s_workGroupNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_workGroupNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_unroll;\n" % kernelName
+ self.incStr += "extern const char * const %s_src;\n" % kernelName
+ self.cppStr += "#include \"%s%s_src.cpp\"\n" % (Common.getRelativeKernelSourcePath(), kernelName)
+ kernelName = kernel.getRowName()
+ self.incStr += "extern const unsigned int %s_workGroupNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_workGroupNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_unroll;\n" % kernelName
+ self.incStr += "extern const char * const %s_src;\n" % kernelName
+ self.cppStr += "#include \"%s%s_src.cpp\"\n" % (Common.getRelativeKernelSourcePath(), kernelName )
+ kernelName = kernel.getColName()
+ self.incStr += "extern const unsigned int %s_workGroupNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_workGroupNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_unroll;\n" % kernelName
+ self.incStr += "extern const char * const %s_src;\n" % kernelName
+ self.cppStr += "#include \"%s%s_src.cpp\"\n" % (Common.getRelativeKernelSourcePath(), kernelName)
+ kernelName = kernel.getCornerName()
+ self.incStr += "extern const unsigned int %s_workGroupNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_workGroupNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumRows;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_microTileNumCols;\n" % kernelName
+ self.incStr += "extern const unsigned int %s_unroll;\n" % kernelName
+ self.incStr += "extern const char * const %s_src;\n" % kernelName
+ self.cppStr += "#include \"%s%s_src.cpp\"\n" % (Common.getRelativeKernelSourcePath(), kernelName)
+
+ self.incFile.write( self.incStr )
+ self.incStr = ""
+ self.cppFile.write( self.cppStr )
+ self.cppStr = ""
+
+ def writeToFile(self):
+ self.incFile.write( self.incStr )
+ self.incFile.write( "\n#endif\n" )
+ self.incFile.close()
+ self.cppFile.write( self.cppStr )
+ self.cppFile.close()
+
+
+################################################################################
+# BINC - Kernel Binary Includes
+################################################################################
+class KernelBinaryIncludes:
+
+ ##############################################################################
+ # BINC - default constructor
+ ##############################################################################
+ def __init__(self):
+ self.incFileName = Common.getIncludePath() + "AutoGemmKernelBinaries.h"
+ self.incFile = open(self.incFileName, "w")
+ self.incFile.write( Common.getAutoGemmHeader() )
+ self.incStr = ""
+ self.incStr += "#include <cstddef>\n"
+ self.incStr += "\n#ifndef AUTOGEMM_KERNEL_BINARIES_H\n"
+ self.incStr += "#define AUTOGEMM_KERNEL_BINARIES_H\n"
+ self.incStr += "\n"
+
+ self.cppFileName = Common.getIncludePath() + "AutoGemmKernelBinaries.cpp"
+ self.cppFile = open(self.cppFileName, "w")
+ self.cppFile.write( Common.getAutoGemmHeader() )
+ self.cppStr = ""
+ self.cppStr += "#include \"%sAutoGemmKernelBinaries.h\"\n" % Common.getRelativeIncludePath()
+ self.cppStr += "\n"
+ self.cppStr += "#ifdef AUTOGEMM_USE_PRE_COMPILED_KERNELS\n"
+ self.cppStr += "#include \"%sAutoGemmKernelBinariesPreCompiled.h\"\n" % Common.getRelativeKernelBinaryPath()
+ self.cppStr += "#endif\n"
+ self.cppStr += "\n"
+
+ def addKernel(self, kernel):
+ kernelName = kernel.getName()
+ self.incStr += "extern unsigned char *%s_bin;\n" % kernelName
+ self.incStr += "extern size_t %s_binSize;\n" % kernelName
+ self.cppStr += "#ifndef KERNEL_" + kernelName.upper() + "_BIN_CPP\n"
+ self.cppStr += "unsigned char *%s_bin = 0;\n" % kernelName
+ self.cppStr += " size_t %s_binSize = 0;\n" % kernelName
+ self.cppStr += "#else\n"
+ self.cppStr += "#pragma message(\"AutoGemmKernelBinaries.cpp: %s was pre-compiled.\")\n" % kernelName
+ self.cppStr += "#endif\n"
+
+ kernelName = kernel.getRowName()
+ self.incStr += "extern unsigned char *%s_bin;\n" % kernelName
+ self.incStr += "extern size_t %s_binSize;\n" % kernelName
+ self.cppStr += "#ifndef KERNEL_" + kernelName.upper() + "_BIN_CPP\n"
+ self.cppStr += "unsigned char *%s_bin = 0;\n" % kernelName
+ self.cppStr += " size_t %s_binSize = 0;\n" % kernelName
+ self.cppStr += "#else\n"
+ self.cppStr += "#pragma message(\"AutoGemmKernelBinaries.cpp: %s was pre-compiled.\")\n" % kernelName
+ self.cppStr += "#endif\n"
+
+ kernelName = kernel.getColName()
+ self.incStr += "extern unsigned char *%s_bin;\n" % kernelName
+ self.incStr += "extern size_t %s_binSize;\n" % kernelName
+ self.cppStr += "#ifndef KERNEL_" + kernelName.upper() + "_BIN_CPP\n"
+ self.cppStr += "unsigned char *%s_bin = 0;\n" % kernelName
+ self.cppStr += " size_t %s_binSize = 0;\n" % kernelName
+ self.cppStr += "#else\n"
+ self.cppStr += "#pragma message(\"AutoGemmKernelBinaries.cpp: %s was pre-compiled.\")\n" % kernelName
+ self.cppStr += "#endif\n"
+
+ kernelName = kernel.getCornerName()
+ self.incStr += "extern unsigned char *%s_bin;\n" % kernelName
+ self.incStr += "extern size_t %s_binSize;\n" % kernelName
+ self.cppStr += "#ifndef KERNEL_" + kernelName.upper() + "_BIN_CPP\n"
+ self.cppStr += "unsigned char *%s_bin = 0;\n" % kernelName
+ self.cppStr += " size_t %s_binSize = 0;\n" % kernelName
+ self.cppStr += "#else\n"
+ self.cppStr += "#pragma message(\"AutoGemmKernelBinaries.cpp: %s was pre-compiled.\")\n" % kernelName
+ self.cppStr += "#endif\n"
+
+ self.incFile.write( self.incStr )
+ self.incStr = ""
+ self.cppFile.write( self.cppStr )
+ self.cppStr = ""
+
+ def writeToFile(self):
+ self.incFile.write( self.incStr )
+ self.incFile.write( "\n#endif\n" )
+ self.incFile.close()
+ self.cppFile.write( self.cppStr )
+ self.cppFile.close()
+
+
+################################################################################
+# CINC - ClKernel Includes
+################################################################################
+class ClKernelIncludes:
+
+ ##############################################################################
+ # CINC - default constructor
+ ##############################################################################
+ def __init__(self):
+ self.incName = Common.getIncludePath() + "AutoGemmClKernels.h"
+ self.incFile = open(self.incName, "w")
+ self.incFile.write( Common.getAutoGemmHeader() )
+ self.incStr = "#ifndef AUTOGEMM_CL_KERNELS_H\n"
+ self.incStr += "#define AUTOGEMM_CL_KERNELS_H\n"
+ self.incStr += "#include \"CL/cl.h\"\n"
+ self.incStr += "\n"
+
+ self.cppName = Common.getIncludePath() + "AutoGemmClKernels.cpp"
+ self.cppFile = open(self.cppName, "w")
+ self.cppFile.write( Common.getAutoGemmHeader() )
+ self.cppStr = "#include \"CL/cl.h\"\n"
+ self.cppStr += "\n"
+
+ def addKernel(self, kernel):
+ kernelName = kernel.getName()
+ self.incStr += "extern cl_kernel %s_clKernel;\n" % kernelName
+ self.cppStr += "cl_kernel %s_clKernel = NULL;\n" % kernelName
+ kernelName = kernel.getRowName()
+ self.incStr += "extern cl_kernel %s_clKernel;\n" % kernelName
+ self.cppStr += "cl_kernel %s_clKernel = NULL;\n" % kernelName
+ kernelName = kernel.getColName()
+ self.incStr += "extern cl_kernel %s_clKernel;\n" % kernelName
+ self.cppStr += "cl_kernel %s_clKernel = NULL;\n" % kernelName
+ kernelName = kernel.getCornerName()
+ self.incStr += "extern cl_kernel %s_clKernel;\n" % kernelName
+ self.cppStr += "cl_kernel %s_clKernel = NULL;\n" % kernelName
+
+ self.incFile.write( self.incStr )
+ self.incStr = ""
+ self.cppFile.write( self.cppStr )
+ self.cppStr = ""
+
+ def writeToFile(self):
+ self.incFile.write( self.incStr )
+ self.incFile.write( "\n#endif\n" )
+ self.incFile.close()
+ self.cppFile.write( self.cppStr )
+ self.cppFile.close()
+
+
+################################################################################
+# KSBO - Kernel Source Build Options
+################################################################################
+class KernelSourceBuildOptions:
+
+ ##############################################################################
+ # KSBO - default constructor
+ ##############################################################################
+ def __init__(self):
+ self.incName = Common.getIncludePath() + "AutoGemmKernelBuildOptionsSource.h"
+ self.incFile = open(self.incName, "w")
+ self.incFile.write( Common.getAutoGemmHeader() )
+ self.incStr = "#ifndef AUTOGEMM_KERNEL_SOURCE_BUILD_OPTIONS_H\n"
+ self.incStr += "#define AUTOGEMM_KERNEL_SOURCE_BUILD_OPTIONS_H\n"
+ self.incStr += "\n"
+
+ self.cppName = Common.getIncludePath() + "AutoGemmKernelBuildOptionsSource.cpp"
+ self.cppFile = open(self.cppName, "w")
+ self.cppFile.write( Common.getAutoGemmHeader() )
+ self.cppStr = ""
+ self.cppStr += "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBuildOptionsSource.h\"\n"
+
+ def addKernel(self, kernel):
+ kernelName = kernel.getName()
+ self.incStr += "extern const char * const %s_srcBuildOptions;\n" \
+ % kernelName
+ self.cppStr += "const char * const %s_srcBuildOptions = \"-cl-std=CL%s\";\n" \
+ % (kernelName, Common.getClCompilerVersion() )
+
+ self.incFile.write( self.incStr )
+ self.incStr = ""
+ self.cppFile.write( self.cppStr )
+ self.cppStr = ""
+
+ def writeToFile(self):
+ self.incFile.write( self.incStr )
+ self.incFile.write( "\n#endif\n" )
+ self.incFile.close()
+
+ self.cppFile.write( self.cppStr )
+ self.cppFile.close()
+
+
+################################################################################
+# KBSO - Kernel Binary Build Options
+################################################################################
+class KernelBinaryBuildOptions:
+
+ ##############################################################################
+ # KBSO - default constructor
+ ##############################################################################
+ def __init__(self):
+ self.incName = Common.getIncludePath() + "AutoGemmKernelBuildOptionsBinary.h"
+ self.incFile = open(self.incName, "w")
+ self.incFile.write( Common.getAutoGemmHeader() )
+ self.incStr = "#ifndef AUTOGEMM_KERNEL_BINARY_BUILD_OPTIONS_H\n"
+ self.incStr += "#define AUTOGEMM_KERNEL_BINARY_BUILD_OPTIONS_H\n"
+ self.incStr += "\n"
+
+ self.cppName = Common.getIncludePath() + "AutoGemmKernelBuildOptionsBinary.cpp"
+ self.cppFile = open(self.cppName, "w")
+ self.cppFile.write( Common.getAutoGemmHeader() )
+ self.cppStr = ""
+ self.cppStr += "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBuildOptionsBinary.h\"\n"
+
+ def addKernel(self, kernel):
+ kernelName = kernel.getName()
+ self.incStr += "extern const char * const %s_binBuildOptions;\n" % kernelName
+ self.cppStr += "const char * const %s_binBuildOptions = \"-cl-std=CL%s\";\n" % (kernelName, Common.getClCompilerVersion() )
+
+ self.incFile.write( self.incStr )
+ self.incStr = ""
+ self.cppFile.write( self.cppStr )
+ self.cppStr = ""
+
+ def writeToFile(self):
+ self.incFile.write( self.incStr )
+ self.incFile.write( "\n#endif\n" )
+ self.incFile.close()
+ self.cppFile.write( self.cppStr )
+ self.cppFile.close()
+
+
+################################################################################
+# CPPKE - Cpp Kernel enumeration
+################################################################################
+class CppKernelEnumeration:
+
+ ##############################################################################
+ # CPPKE - default constructor
+ ##############################################################################
+ def __init__(self):
+ self.fileName = Common.getIncludePath() + "AutoGemmKernelEnumeration.h"
+ self.kernelStr = ""
+ self.tileStr = ""
+ self.nonTileStr = ""
+ self.kernelCount = 0
+ self.tileCount = 0
+ self.nonTileCount = 0
+ self.precision = ""
+ self.precisionInitialized = False
+
+ def newPrecision(self, precision):
+ if self.precisionInitialized:
+ self.kernelStr += "};\n"
+ self.kernelStr += "const unsigned int %sgemmNumKernels = %d;\n\n" \
+ % (self.precision, self.kernelCount)
+ self.tileStr += "};\n"
+ self.tileStr += "const unsigned int %sgemmNumTiles = %d;\n\n" \
+ % (self.precision, self.tileCount)
+ self.nonTileStr += "};\n"
+ self.nonTileStr += "const unsigned int %sgemmNumNonTiles = %d;\n\n" \
+ % (self.precision, self.nonTileCount)
+ self.precisionInitialized = True
+ self.precision = precision
+
+ self.kernelStr += "// order, transA, transB, beta, macroTileNumRows, macroTileNumCols, unroll, mSpill, nSpill\n"
+ self.kernelStr += "unsigned int " + precision + "gemmKernelEnumeration[][9] = {\n"
+
+ self.tileStr += "// macroTileNumRows, macroTileNumCols, unroll\n"
+ self.tileStr += "unsigned int " + precision + "gemmTileEnumeration[][3] = {\n"
+
+ self.nonTileStr += "// order, transA, transB, beta\n"
+ self.nonTileStr += "unsigned int " + precision + "gemmNonTileEnumeration[][4] = {\n"
+ self.tileCount = 0
+ self.nonTileCount = 0
+ self.kernelCount = 0
+
+ def addTile(self, tile):
+ self.tileStr += " { %3u, %3u, %1u },\n" % ( \
+ tile.macroTileNumRows, \
+ tile.macroTileNumCols, \
+ tile.unroll )
+ self.tileCount += 1
+
+ def addNonTile(self, nonTile):
+ self.nonTileStr += " { %1u, %1u, %1u, %1u },\n" % ( \
+ 1 if nonTile.order=="clblasColumnMajor" else 0, \
+ 0 if nonTile.transA=="N" else 1 if nonTile.transA=="T" else 2 , \
+ 0 if nonTile.transB=="N" else 1 if nonTile.transB=="T" else 2, \
+ 1 if nonTile.beta>0 else 0 )
+ self.nonTileCount += 1
+
+ def addKernel(self, kernel):
+ # 6) list to add to ktest for automated kernel testing
+ for mSpill in range(0, 2):
+ for nSpill in range(0, 2):
+ self.kernelStr += " { %1u, %1u, %1u, %1u, %3u, %3u, %2u, %1u, %1u },\n" % ( \
+ 1 if kernel.order=="clblasColumnMajor" else 0, \
+ 0 if kernel.transA=="N" else 1 if kernel.transA=="T" else 2 , \
+ 0 if kernel.transB=="N" else 1 if kernel.transB=="T" else 2, \
+ 1 if kernel.beta>0 else 0, \
+ kernel.macroTileNumRows, \
+ kernel.macroTileNumCols, \
+ kernel.unroll, \
+ mSpill, \
+ nSpill )
+ self.kernelCount += 4
+
+ def writeToFile(self):
+ self.kernelStr += "};\n"
+ self.kernelStr += "const unsigned int %sgemmNumKernels = %d;\n" % (self.precision, self.kernelCount)
+ self.tileStr += "};\n"
+ self.tileStr += "const unsigned int %sgemmNumTiles = %d;\n" % (self.precision, self.tileCount)
+ self.nonTileStr += "};\n"
+ self.nonTileStr += "const unsigned int %sgemmNumNonTiles = %d;\n" % (self.precision, self.nonTileCount)
+ incFile = open(self.fileName, "w")
+ incFile.write( Common.getAutoGemmHeader() )
+ incFile.write( self.tileStr )
+ incFile.write( "\n\n" )
+ incFile.write( self.nonTileStr )
+ incFile.write( "\n\n" )
+ incFile.write( self.kernelStr )
+ incFile.close()
+
+################################################################################
+# Write Includes
+################################################################################
+def writeIncludes():
+ print "AutoGemm.py: Generating include files."
+ if not os.path.exists( Common.getIncludePath() ):
+ os.makedirs( Common.getIncludePath() )
+
+ kernelSourceIncludes = KernelSourceIncludes()
+ kernelBinaryIncludes = KernelBinaryIncludes()
+ clKernelIncludes = ClKernelIncludes()
+ kernelSourceBuildOptions = KernelSourceBuildOptions()
+ kernelBinaryBuildOptions = KernelBinaryBuildOptions()
+ cppKernelEnumeration = CppKernelEnumeration()
+
+
+ # for each precision
+ kernel = KernelParameters.KernelParameters()
+ for precision in AutoGemmParameters.precisions:
+ kernel.precision = precision
+ cppKernelEnumeration.newPrecision(precision)
+
+ # valid tiles for this precision
+ tiles = AutoGemmParameters.getTilesForPrecision(precision)
+
+ # add tiles for this precision to Cpp
+ for tile in tiles:
+ cppKernelEnumeration.addTile(tile)
+
+ # for non tile parameters
+ for order in AutoGemmParameters.orders:
+ kernel.order = order
+ for transA in AutoGemmParameters.transposes[precision]:
+ kernel.transA = transA
+ for transB in AutoGemmParameters.transposes[precision]:
+ kernel.transB = transB
+ for beta in AutoGemmParameters.betas:
+ kernel.beta = beta
+
+ # add this nonTile combo for this precision to Cpp
+ cppKernelEnumeration.addNonTile(kernel)
+
+ # for tile parameters
+ for tile in tiles:
+ kernel.useTile(tile)
+ kernelSourceIncludes.addKernel(kernel)
+ kernelBinaryIncludes.addKernel(kernel)
+ kernelSourceBuildOptions.addKernel(kernel)
+ kernelBinaryBuildOptions.addKernel(kernel)
+ clKernelIncludes.addKernel(kernel)
+ cppKernelEnumeration.addKernel(kernel)
+
+ # save written files
+ kernelSourceIncludes.writeToFile()
+ kernelBinaryIncludes.writeToFile()
+ clKernelIncludes.writeToFile()
+ kernelSourceBuildOptions.writeToFile()
+ kernelBinaryBuildOptions.writeToFile()
+ cppKernelEnumeration.writeToFile()
+
+
+
+################################################################################
+# Main
+################################################################################
+if __name__ == "__main__":
+ if len(sys.argv) == 2:
+ Common.setOutputPath(sys.argv[1])
+ else:
+ print "Warning: No output path specified; default is working directory."
+ writeIncludes()
+
diff --git a/src/library/blas/AutoGemm/KernelOpenCL.py b/src/library/blas/AutoGemm/KernelOpenCL.py
new file mode 100644
index 0000000..0b671f0
--- /dev/null
+++ b/src/library/blas/AutoGemm/KernelOpenCL.py
@@ -0,0 +1,549 @@
+import os
+import sys
+import copy
+import Common
+import KernelParameters
+import AutoGemmParameters
+
+
+##############################################################################
+# Make OpenCL Kernel String
+##############################################################################
+def makeOpenCLKernelString(kernel):
+ endLine = "\\n\"\n\""
+
+ ####################################
+ # parameters valid?
+ if kernel.isValid() == False:
+ return kernel.getName() + " invalid"
+
+ ####################################
+ # initializations
+ kStr = ""
+ kStr += endLine
+ kStr += "/* %s */" % kernel.getName()
+ kStr += endLine
+
+ ####################################
+ # kernel parameters
+ kStr += endLine
+ kStr += "/* kernel parameters */" + endLine
+ #if kernel.order == "clblasColumnMajor":
+ # kStr += "#define COLUMN_MAJOR 1" + endLine
+ #else:
+ # kStr += "#define COLUMN_MAJOR 0" + endLine
+ #if kernel.transA == "T":
+ # kStr += "#define TRANSPOSE_A 1" + endLine
+ #else:
+ # kStr += "#define TRANSPOSE_A 0" + endLine
+ #if kernel.transB == "T":
+ # kStr += "#define TRANSPOSE_B 1" + endLine
+ #else:
+ # kStr += "#define TRANSPOSE_B 0" + endLine
+ #kStr += "" + endLine
+ kStr += "#define WG_NUM_ROWS %d%s" % (kernel.workGroupNumRows, endLine )
+ kStr += "#define WG_NUM_COLS %d%s" % (kernel.workGroupNumCols, endLine )
+ kStr += "#define MICRO_TILE_NUM_ROWS %d%s" % (kernel.microTileNumRows, endLine )
+ kStr += "#define MICRO_TILE_NUM_COLS %d%s" % (kernel.microTileNumCols, endLine )
+ kStr += "#define MACRO_TILE_NUM_ROWS %s%s" % ((kernel.workGroupNumRows * kernel.microTileNumRows), endLine )
+ kStr += "#define MACRO_TILE_NUM_COLS %s%s" % ((kernel.workGroupNumCols * kernel.microTileNumCols), endLine )
+ kStr += "#define NUM_UNROLL_ITER %s%s" % (kernel.unroll, endLine )
+ kStr += "" + endLine
+ kStr += "#define LOCAL_ROW_PAD %s%s" % (kernel.localRowPad, endLine)
+ kStr += "#define LOCAL_COL_PAD %s%s" % (kernel.localColPad, endLine)
+
+ ####################################
+ # global memory indices
+ # A
+ kStr += endLine
+ kStr += "/* global memory indices */" + endLine
+ if (kernel.order=="clblasColumnMajor")==(kernel.transA=="N"):
+ kStr += "#define GET_GLOBAL_INDEX_A(ROW,COL) ((COL)*lda+(ROW))" + endLine
+ else:
+ kStr += "#define GET_GLOBAL_INDEX_A(ROW,COL) ((ROW)*lda+(COL))" + endLine
+ # B
+ if (kernel.order=="clblasColumnMajor")==(kernel.transB=="N"):
+ kStr += "#define GET_GLOBAL_INDEX_B(ROW,COL) ((COL)*ldb+(ROW))" + endLine
+ else:
+ kStr += "#define GET_GLOBAL_INDEX_B(ROW,COL) ((ROW)*ldb+(COL))" + endLine
+ # C
+ if (kernel.order=="clblasColumnMajor"):
+ kStr += "#define GET_GLOBAL_INDEX_C(ROW,COL) ((COL)*ldc+(ROW))" + endLine
+ else:
+ kStr += "#define GET_GLOBAL_INDEX_C(ROW,COL) ((ROW)*ldc+(COL))" + endLine
+
+ ####################################
+ # local memory indices
+ # A
+ kStr += endLine
+ kStr += "/* local memory indices */" + endLine
+ kStr += "#define GET_LOCAL_INDEX_A(ROW,COL) ((ROW) + (COL)*((MACRO_TILE_NUM_ROWS)+(LOCAL_COL_PAD)) )" + endLine
+ # B
+ kStr += "#define GET_LOCAL_INDEX_B(ROW,COL) ((COL) + (ROW)*((MACRO_TILE_NUM_COLS)+(LOCAL_ROW_PAD)) )" + endLine
+
+ ####################################
+ # data types
+ kStr += endLine
+ kStr += "/* data types */" + endLine
+ kStr += "#define DATA_TYPE_STR %s%s" \
+ % (Common.openclDataType[kernel.precision], endLine)
+ if kernel.precision=="s" or kernel.precision=="d":
+ # real arithmetic
+ kStr += "#define TYPE_MAD(MULA,MULB,DST) DST = mad(MULA,MULB,DST);" + endLine
+ if kernel.beta==1:
+ kStr += "#define TYPE_MAD_WRITE(DST,ALPHA,REG,BETA) DST = (ALPHA)*(REG) + (BETA)*(DST);" + endLine
+ else:
+ kStr += "#define TYPE_MAD_WRITE(DST,ALPHA,REG,BETA) DST = (ALPHA)*(REG);" + endLine
+
+ else:
+ # complex arithmetic
+ if kernel.transA!="C" and kernel.transB!="C":
+ # neither conjugate
+ kStr += (
+ "#define TYPE_MAD(MULA,MULB,DST) \\\\" + endLine +
+ " DST.s0 = mad( MULA.s0, MULB.s0, DST.s0 ); \\\\" + endLine +
+ " DST.s0 = mad( -MULA.s1, MULB.s1, DST.s0 ); \\\\" + endLine +
+ " DST.s1 = mad( MULA.s0, MULB.s1, DST.s1 ); \\\\" + endLine +
+ " DST.s1 = mad( MULA.s1, MULB.s0, DST.s1 );" + endLine )
+ elif kernel.transA=="C" and kernel.transB!="C":
+ # A conjugate (negate imaginary A.s1)
+ kStr += (
+ "#define TYPE_MAD(MULA,MULB,DST) \\\\" + endLine +
+ " DST.s0 = mad( MULA.s0, MULB.s0, DST.s0 ); \\\\" + endLine +
+ " DST.s0 = mad( MULA.s1, MULB.s1, DST.s0 ); \\\\" + endLine +
+ " DST.s1 = mad( MULA.s0, MULB.s1, DST.s1 ); \\\\" + endLine +
+ " DST.s1 = mad( -MULA.s1, MULB.s0, DST.s1 );" + endLine )
+ elif kernel.transA!="C" and kernel.transB=="C":
+ # B conjugate (negate imaginary B.s1)
+ kStr += (
+ "#define TYPE_MAD(MULA,MULB,DST) \\\\" + endLine +
+ " DST.s0 = mad( MULA.s0, MULB.s0, DST.s0 ); \\\\" + endLine +
+ " DST.s0 = mad( -MULA.s1, -MULB.s1, DST.s0 ); \\\\" + endLine +
+ " DST.s1 = mad( MULA.s0, -MULB.s1, DST.s1 ); \\\\" + endLine +
+ " DST.s1 = mad( MULA.s1, MULB.s0, DST.s1 );" + endLine )
+ else:
+ # A & B conjugate (negate imaginary .s1)
+ kStr += (
+ "#define TYPE_MAD(MULA,MULB,DST) \\\\" + endLine +
+ " DST.s0 = mad( MULA.s0, MULB.s0, DST.s0 ); \\\\" + endLine +
+ " DST.s0 = mad( MULA.s1, -MULB.s1, DST.s0 ); \\\\" + endLine +
+ " DST.s1 = mad( MULA.s0, -MULB.s1, DST.s1 ); \\\\" + endLine +
+ " DST.s1 = mad( -MULA.s1, MULB.s0, DST.s1 );" + endLine )
+ if kernel.beta==1:
+ kStr += (
+ "#define TYPE_MAD_WRITE( DST, ALPHA, REG, BETA ) \\\\" + endLine +
+ " /* (1) */ \\\\" + endLine +
+ " type_mad_tmp = REG.s0; \\\\" + endLine +
+ " REG.s0 *= ALPHA.s0; \\\\" + endLine +
+ " REG.s0 = mad( -ALPHA.s1, REG.s1, REG.s0 ); \\\\" + endLine +
+ " REG.s1 *= ALPHA.s0; \\\\" + endLine +
+ " REG.s1 = mad( ALPHA.s1, type_mad_tmp, REG.s1 ); \\\\" + endLine +
+ " /* (2) */ \\\\" + endLine +
+ " REG.s0 = mad( BETA.s0, DST.s0, REG.s0 ); \\\\" + endLine +
+ " REG.s0 = mad( -BETA.s1, DST.s1, REG.s0 ); \\\\" + endLine +
+ " REG.s1 = mad( BETA.s1, DST.s0, REG.s1 ); \\\\" + endLine +
+ " REG.s1 = mad( BETA.s0, DST.s1, REG.s1 ); \\\\" + endLine +
+ " /* (3) */ \\\\" + endLine +
+ " DST = REG;" + endLine )
+ else:
+ kStr += (
+ "#define TYPE_MAD_WRITE( DST, ALPHA, REG, BETA ) \\\\" + endLine +
+ " /* (1) */ \\\\" + endLine +
+ " type_mad_tmp = REG.s0; \\\\" + endLine +
+ " REG.s0 *= ALPHA.s0; \\\\" + endLine +
+ " REG.s0 = mad( -ALPHA.s1, REG.s1, REG.s0 ); \\\\" + endLine +
+ " REG.s1 *= ALPHA.s0; \\\\" + endLine +
+ " REG.s1 = mad( ALPHA.s1, type_mad_tmp, REG.s1 ); \\\\" + endLine +
+ " /* (2) */ \\\\" + endLine +
+ " REG.s0 = mad( BETA.s0, DST.s0, REG.s0 ); \\\\" + endLine +
+ " REG.s0 = mad( -BETA.s1, DST.s1, REG.s0 ); \\\\" + endLine +
+ " REG.s1 = mad( BETA.s1, DST.s0, REG.s1 ); \\\\" + endLine +
+ " REG.s1 = mad( BETA.s0, DST.s1, REG.s1 ); \\\\" + endLine +
+ " /* (3) */ \\\\" + endLine +
+ " DST = REG;" + endLine )
+
+ ####################################
+ # micro-tile
+ kStr += endLine
+ kStr += "/* %dx%d micro-tile */%s" % (kernel.microTileNumRows, kernel.microTileNumCols, endLine)
+ kStr += "#define MICRO_TILE \\\\" + endLine
+ for a in range(0, kernel.microTileNumRows):
+ kStr += " rA[%d] = localA[offA + %d*WG_NUM_ROWS]; \\\\%s" % (a, a, endLine)
+ for b in range(0, kernel.microTileNumCols):
+ kStr += " rB[%d] = localB[offB + %d*WG_NUM_COLS]; \\\\%s" % (b, b, endLine)
+ kStr += " offA += (MACRO_TILE_NUM_ROWS+LOCAL_COL_PAD); \\\\" + endLine
+ kStr += " offB += (MACRO_TILE_NUM_COLS+LOCAL_ROW_PAD); \\\\" + endLine
+ for a in range(0, kernel.microTileNumRows):
+ for b in range(0, kernel.microTileNumCols):
+ kStr += " TYPE_MAD(rA[%d],rB[%d],rC[%d][%d]); \\\\%s" % (a, b, a, b, endLine)
+ kStr += " mem_fence(CLK_LOCAL_MEM_FENCE);" + endLine
+ kStr += endLine
+
+ ####################################
+ # function signature
+ ####################################
+ kStr += "__attribute__((reqd_work_group_size(WG_NUM_COLS,WG_NUM_ROWS,1)))" + endLine
+ kStr += "__kernel void %s" % ( kernel.getName() )
+ kStr += "(" + endLine
+ # arguments
+ kStr += (
+ " __global DATA_TYPE_STR const * restrict A," + endLine +
+ " __global DATA_TYPE_STR const * restrict B," + endLine +
+ " __global DATA_TYPE_STR * C," + endLine +
+ " DATA_TYPE_STR const alpha," + endLine +
+ " DATA_TYPE_STR const beta," + endLine +
+ " uint const M," + endLine +
+ " uint const N," + endLine +
+ " uint const K," + endLine +
+ " uint const lda," + endLine +
+ " uint const ldb," + endLine +
+ " uint const ldc," + endLine +
+ " uint const offsetA," + endLine +
+ " uint const offsetB," + endLine +
+ " uint const offsetC" + endLine +
+ ") {" + endLine )
+
+ ####################################
+ # apply offsets
+ kStr += endLine
+ kStr += (
+ " /* apply offsets */" + endLine +
+ " A += offsetA;" + endLine +
+ " B += offsetB;" + endLine +
+ " C += offsetC;" + endLine )
+
+ ####################################
+ # allocate registers
+ kStr += endLine
+ kStr += (
+ " /* allocate registers */" + endLine +
+ " DATA_TYPE_STR rC[MICRO_TILE_NUM_ROWS][MICRO_TILE_NUM_COLS] = {0};" + endLine +
+ " DATA_TYPE_STR rA[MICRO_TILE_NUM_ROWS];" + endLine +
+ " DATA_TYPE_STR rB[MICRO_TILE_NUM_COLS];" + endLine )
+
+ ####################################
+ # allocate local memory
+ kStr += endLine
+ kStr += (
+ " /* allocate local memory */" + endLine +
+ " __local DATA_TYPE_STR localA[NUM_UNROLL_ITER*(MACRO_TILE_NUM_ROWS+LOCAL_COL_PAD)];" + endLine +
+ " __local DATA_TYPE_STR localB[NUM_UNROLL_ITER*(MACRO_TILE_NUM_COLS+LOCAL_ROW_PAD)];" + endLine )
+
+ ####################################
+ # work item indices
+ kStr += endLine
+ kStr += " /* work item indices */" + endLine
+ if kernel.isRowKernel():
+ kStr += " uint groupRow = M / " + str(kernel.workGroupNumRows*kernel.microTileNumRows) + "; // last row" + endLine
+ else:
+ kStr += " uint groupRow = get_group_id(0);" + endLine
+ if kernel.isColKernel():
+ kStr += " uint groupCol = N / " + str(kernel.workGroupNumCols*kernel.microTileNumCols) + "; // last column" + endLine
+ else:
+ kStr += " uint groupCol = get_group_id(1);" + endLine
+
+ ####################################
+ # z-order - TODO doesn't improve caching, only lowers occupancy
+ if False:
+ kStr += (
+ " // convert work-group order to z-order" + endLine +
+ " unsigned int morton = get_group_id(1) * get_num_groups(0) + get_group_id(0);" + endLine +
+ " groupRow = morton;" + endLine +
+ " groupCol = ( groupRow >> 1 );" + endLine +
+ " groupRow &= 0x55555555;" + endLine +
+ " groupCol &= 0x55555555;" + endLine +
+ " groupRow |= ( groupRow >> 1 );" + endLine +
+ " groupCol |= ( groupCol >> 1 );" + endLine +
+ " groupRow &= 0x33333333;" + endLine +
+ " groupCol &= 0x33333333;" + endLine +
+ " groupRow |= ( groupRow >> 2 );" + endLine +
+ " groupCol |= ( groupCol >> 2 );" + endLine +
+ " groupRow &= 0x0f0f0f0f;" + endLine +
+ " groupCol &= 0x0f0f0f0f;" + endLine +
+ " groupRow |= ( groupRow >> 4 );" + endLine +
+ " groupCol |= ( groupCol >> 4 );" + endLine +
+ " groupRow &= 0x00ff00ff;" + endLine +
+ " groupCol &= 0x00ff00ff;" + endLine +
+ " groupRow |= ( groupRow >> 8 );" + endLine +
+ " groupCol |= ( groupCol >> 8 );" + endLine +
+ " groupRow &= 0x0000ffff;" + endLine +
+ " groupCol &= 0x0000ffff;" + endLine + endLine
+ )
+
+ kStr += (
+ " uint localRow = get_local_id(0);" + endLine +
+ " uint localCol = get_local_id(1);" + endLine +
+ " uint localSerial = localRow + localCol*WG_NUM_ROWS;" + endLine )
+
+ ####################################
+ # global indices being loaded
+ kStr += endLine
+ kStr += " /* global indices being loaded */" + endLine
+ if (kernel.order=="clblasColumnMajor")==(kernel.transA=="N"):
+ kStr += (
+ "#define globalARow(LID) (groupRow*MACRO_TILE_NUM_ROWS + (localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)%MACRO_TILE_NUM_ROWS)" + endLine +
+ "#define globalACol(LID) ((localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)/MACRO_TILE_NUM_ROWS)" + endLine )
+ else:
+ kStr += (
+ "#define globalARow(LID) (groupRow*MACRO_TILE_NUM_ROWS + (localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)/NUM_UNROLL_ITER)" + endLine +
+ "#define globalACol(LID) ((localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)%NUM_UNROLL_ITER)" + endLine )
+
+ if (kernel.order=="clblasColumnMajor")==(kernel.transB=="N"):
+ kStr += (
+ "#define globalBRow(LID) ((localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)%NUM_UNROLL_ITER)" + endLine +
+ "#define globalBCol(LID) (groupCol*MACRO_TILE_NUM_COLS + (localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)/NUM_UNROLL_ITER)" + endLine )
+ else:
+ kStr += (
+ "#define globalBRow(LID) ((localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)/MACRO_TILE_NUM_COLS)" + endLine +
+ "#define globalBCol(LID) (groupCol*MACRO_TILE_NUM_COLS + (localSerial+(LID)*WG_NUM_ROWS*WG_NUM_COLS)%MACRO_TILE_NUM_COLS)" + endLine )
+
+ #kStr += (
+ # " A += GET_GLOBAL_INDEX_A( globalARow, globalACol );" + endLine +
+ # " B += GET_GLOBAL_INDEX_B( globalBRow, globalBCol );" + endLine )
+
+ ####################################
+ # loop over k
+ kStr += endLine
+ kStr += (
+ " /* loop over k */" + endLine +
+ " uint block_k = K / NUM_UNROLL_ITER;" + endLine +
+ " do {" + endLine )
+
+ ####################################
+ # local indices being written
+ kStr += endLine
+ kStr += " /* local indices being written */" + endLine
+ if (kernel.order=="clblasColumnMajor")==(kernel.transA=="N"):
+ kStr += (
+ "#define localARow (localSerial % MACRO_TILE_NUM_ROWS)" + endLine +
+ "#define localACol (localSerial / MACRO_TILE_NUM_ROWS)" + endLine +
+ "#define localAStride (WG_NUM_ROWS*WG_NUM_COLS)" + endLine )
+ else:
+ kStr += (
+ "#define localARow (localSerial / NUM_UNROLL_ITER)" + endLine +
+ "#define localACol (localSerial % NUM_UNROLL_ITER)" + endLine +
+ "#define localAStride (WG_NUM_ROWS*WG_NUM_COLS/NUM_UNROLL_ITER)" + endLine )
+
+ if (kernel.order=="clblasColumnMajor")==(kernel.transB=="N"):
+ kStr += (
+ "#define localBRow ( localSerial % NUM_UNROLL_ITER )" + endLine +
+ "#define localBCol ( localSerial / NUM_UNROLL_ITER )" + endLine +
+ "#define localBStride (WG_NUM_ROWS*WG_NUM_COLS/NUM_UNROLL_ITER)" + endLine )
+ else:
+ kStr += (
+ "#define localBRow ( localSerial / MACRO_TILE_NUM_COLS )" + endLine +
+ "#define localBCol ( localSerial % MACRO_TILE_NUM_COLS )" + endLine +
+ "#define localBStride (WG_NUM_ROWS*WG_NUM_COLS)" + endLine )
+
+
+ kStr += (
+ " __local DATA_TYPE_STR *lA = localA + GET_LOCAL_INDEX_A(localARow, localACol);" + endLine +
+ " __local DATA_TYPE_STR *lB = localB + GET_LOCAL_INDEX_B(localBRow, localBCol);" + endLine +
+ " barrier(CLK_LOCAL_MEM_FENCE);" + endLine )
+
+ ####################################
+ # load global -> local
+ # threads to do loading = (workGroupNumRows*workGroupNumCols)
+ # A elements to be loaded = workGroupNumRows*microTileNumRows*unroll
+ # B elements to be loaded = workGroupNumCols*microTileNumCols*unroll
+ kStr += endLine
+ kStr += " /* load global -> local */" + endLine
+ numALoads = (kernel.workGroupNumRows*kernel.microTileNumRows*kernel.unroll) \
+ / (kernel.workGroupNumRows*kernel.workGroupNumCols)
+ numALoadsR = (kernel.workGroupNumRows*kernel.microTileNumRows*kernel.unroll) \
+ % (kernel.workGroupNumRows*kernel.workGroupNumCols)
+ numBLoads = (kernel.workGroupNumCols*kernel.microTileNumCols*kernel.unroll) \
+ / (kernel.workGroupNumRows*kernel.workGroupNumCols)
+ numBLoadsR = (kernel.workGroupNumCols*kernel.microTileNumCols*kernel.unroll) \
+ % (kernel.workGroupNumRows*kernel.workGroupNumCols)
+
+ # TODO - zeroString for real and complex
+ if kernel.precision == "c":
+ zeroString = "(float2)(0.f, 0.f)"
+ elif kernel.precision == "z":
+ zeroString = "(double2)(0.0, 0.0)"
+ else:
+ zeroString = "0.0"
+ for a in range(0, numALoads):
+ kStr += " lA[ %d*localAStride ] = " % a
+ if kernel.isRowKernel():
+ kStr += "( globalARow(%d) >= M) ? %s : " % ( a, zeroString )
+ kStr += "A[ GET_GLOBAL_INDEX_A( globalARow(%d), globalACol(%d) ) ];%s" % (a, a, endLine)
+ if numALoadsR:
+ kStr += " if ( localSerial + " + str(numALoads) + "*WG_NUM_ROWS*WG_NUM_COLS < (WG_NUM_ROWS*MICRO_TILE_NUM_ROWS*NUM_UNROLL_ITER) ) {" + endLine
+ kStr += " lA[ %d*localAStride ] = " % numALoads
+ if kernel.isRowKernel():
+ kStr += "( globalARow(%d) >= M) ? %s : " % ( numALoads, zeroString )
+ kStr += "A[ GET_GLOBAL_INDEX_A( globalARow(%d), globalACol(%d) ) ];%s" % (numALoads, numALoads, endLine)
+ kStr += " }" + endLine
+
+ for b in range(0, numBLoads):
+ kStr += " lB[ %d*localBStride ] = " % b
+ if kernel.isColKernel():
+ kStr += "( globalBCol(%d) >= N) ? %s : " % ( b, zeroString )
+ kStr += "B[ GET_GLOBAL_INDEX_B( globalBRow(%d), globalBCol(%d) ) ];%s" % (b, b, endLine)
+ if numBLoadsR:
+ kStr += " if ( localSerial + " + str(numBLoads) + "*WG_NUM_ROWS*WG_NUM_COLS < (WG_NUM_COLS*MICRO_TILE_NUM_COLS*NUM_UNROLL_ITER) ) {" + endLine
+ kStr += " lB[ %d*localBStride ] = " % numBLoads
+ if kernel.isColKernel():
+ kStr += "(globalBCol(%d) >= N) ? %s : " % ( numBLoads, zeroString )
+ kStr += "B[ GET_GLOBAL_INDEX_B( globalBRow(%d), globalBCol(%d) ) ];%s" % (numBLoads, numBLoads, endLine)
+ kStr += " }" + endLine
+ kStr += (
+ " barrier(CLK_LOCAL_MEM_FENCE);" + endLine +
+ " uint offA = localRow;" + endLine +
+ " uint offB = localCol;" + endLine )
+
+ ####################################
+ # do mads
+ kStr += endLine
+ kStr += " /* do mads */" + endLine
+ for u in range(0, kernel.unroll):
+ kStr += " MICRO_TILE" + endLine
+
+ ####################################
+ # shift to next k block
+ kStr += endLine
+ kStr += " /* shift to next k block */" + endLine
+ if (kernel.order=="clblasColumnMajor")==(kernel.transA=="N"):
+ kStr += " A += lda*NUM_UNROLL_ITER;" + endLine
+ else:
+ kStr += " A += NUM_UNROLL_ITER;" + endLine
+ if (kernel.order=="clblasColumnMajor")==(kernel.transB=="N"):
+ kStr += " B += NUM_UNROLL_ITER;" + endLine
+ else:
+ kStr += " B += ldb*NUM_UNROLL_ITER;" + endLine
+
+ ####################################
+ # end loop
+ kStr += endLine
+ kStr += " } while (--block_k > 0);" + endLine
+ kStr += endLine
+
+ ####################################
+ # which global Cij index
+ kStr += endLine
+ kStr += " /* which global Cij index */" + endLine
+ kStr += " uint globalCRow = groupRow * MACRO_TILE_NUM_ROWS + localRow;" + endLine
+ kStr += " uint globalCCol = groupCol * MACRO_TILE_NUM_COLS + localCol;" + endLine
+
+ ####################################
+ # write global Cij
+ kStr += endLine
+ kStr += " /* write global Cij */" + endLine
+ if kernel.precision=="c":
+ kStr += " float type_mad_tmp;" + endLine
+ if kernel.precision=="z":
+ kStr += " double type_mad_tmp;" + endLine
+
+ for a in range(0, kernel.microTileNumRows):
+ for b in range(0, kernel.microTileNumCols):
+ if kernel.isRowKernel():
+ kStr += " if (globalCRow+%d*WG_NUM_ROWS < M)" % a
+ if kernel.isColKernel():
+ kStr += " if (globalCCol+%d*WG_NUM_COLS < N)" % b
+ if kernel.isRowKernel() or kernel.isColKernel():
+ kStr += "{"
+ kStr += " TYPE_MAD_WRITE( C[ GET_GLOBAL_INDEX_C( globalCRow+%d*WG_NUM_ROWS, globalCCol+%d*WG_NUM_COLS) ], alpha, rC[%d][%d], beta )" % (a, b, a, b)
+ if kernel.isRowKernel() or kernel.isColKernel():
+ kStr += "}"
+ kStr += endLine
+
+ ####################################
+ # end kernel
+ kStr += endLine
+ kStr += "}" + endLine
+
+ return kStr
+
+
+##############################################################################
+# Write OpenCL kernel to file
+##############################################################################
+def writeOpenCLKernelToFile(kernel):
+ kernelName = kernel.getName()
+ kernelString = makeOpenCLKernelString(kernel)
+ kernelFileName = Common.getKernelSourcePath() + kernelName +"_src.cpp"
+ kernelFile = open(kernelFileName, "w")
+ kernelFile.write( Common.getAutoGemmHeader() )
+ kernelFile.write("#ifndef KERNEL_" + kernelName.upper() + "_SRC_H\n")
+ kernelFile.write("#define KERNEL_" + kernelName.upper() + "_SRC_H\n")
+ kernelFile.write("\n")
+ kernelFile.write("const unsigned int %s_workGroupNumRows = %u;\n" % (kernel.getName(), kernel.workGroupNumRows ) )
+ kernelFile.write("const unsigned int %s_workGroupNumCols = %u;\n" % (kernel.getName(), kernel.workGroupNumCols ) )
+ kernelFile.write("const unsigned int %s_microTileNumRows = %u;\n" % (kernel.getName(), kernel.microTileNumRows ) )
+ kernelFile.write("const unsigned int %s_microTileNumCols = %u;\n" % (kernel.getName(), kernel.microTileNumCols ) )
+ kernelFile.write("const unsigned int %s_unroll = %u;\n" % (kernel.getName(), kernel.unroll) )
+ kernelFile.write("\n")
+ kernelFile.write("const char * const %s_src =\"" % (kernelName) )
+ kernelFile.write(kernelString)
+ kernelFile.write("\";\n")
+ kernelFile.write("\n")
+ kernelFile.write("#else\n")
+ kernelFile.write("#pragma message(\"AutoGemmKernelSources.cpp: %s was overriden by user kernel.\")\n" % kernel.getName() )
+ kernelFile.write("#endif\n")
+ kernelFile.close()
+
+
+##############################################################################
+# Write OpenCL kernel to file
+##############################################################################
+def writeOpenCLKernels():
+
+ if not os.path.exists( Common.getKernelSourcePath() ):
+ os.makedirs( Common.getKernelSourcePath() )
+ if not os.path.exists( Common.getKernelBinaryPath() ):
+ os.makedirs( Common.getKernelBinaryPath() )
+
+ numKernels = 0
+ # for each precision
+ kernel = KernelParameters.KernelParameters()
+ for precision in AutoGemmParameters.precisions:
+ kernel.precision = precision
+
+ # valid tiles for this precision
+ tiles = AutoGemmParameters.getTilesForPrecision(precision)
+
+ # for non tile parameters
+ for order in AutoGemmParameters.orders:
+ kernel.order = order
+ for transA in AutoGemmParameters.transposes[precision]:
+ kernel.transA = transA
+ for transB in AutoGemmParameters.transposes[precision]:
+ kernel.transB = transB
+ for beta in AutoGemmParameters.betas:
+ kernel.beta = beta
+
+ # for tile parameters
+ for tile in tiles:
+ # tile kernel
+ kernel.useTile(tile)
+ writeOpenCLKernelToFile(kernel)
+ # row kernel
+ rowKernel = copy.copy(kernel)
+ rowKernel.macroTileNumRows = 1
+ writeOpenCLKernelToFile(rowKernel)
+ # col kernel
+ colKernel = copy.copy(kernel)
+ colKernel.macroTileNumCols = 1
+ writeOpenCLKernelToFile(colKernel)
+ # corner kernel
+ cornerKernel = copy.copy(kernel)
+ cornerKernel.macroTileNumRows = 1
+ cornerKernel.macroTileNumCols = 1
+ writeOpenCLKernelToFile(cornerKernel)
+ numKernels += 4
+ print "AutoGemm.py: generated %d kernels" % numKernels
+
+
+
+################################################################################
+# Main
+################################################################################
+if __name__ == "__main__":
+ if len(sys.argv) == 2:
+ Common.setOutputPath(sys.argv[1])
+ else:
+ print "Warning: No output path specified; default is working directory."
+ writeOpenCLKernels()
+
diff --git a/src/library/blas/AutoGemm/KernelParameters.py b/src/library/blas/AutoGemm/KernelParameters.py
new file mode 100644
index 0000000..058ae19
--- /dev/null
+++ b/src/library/blas/AutoGemm/KernelParameters.py
@@ -0,0 +1,253 @@
+import copy
+import Common
+
+################################################################################
+# Tile Parameters
+# - parameters which should match matrix system for good performance
+################################################################################
+class TileParameters:
+
+ nameFormatTile = "MX%03d_NX%03d_KX%02d"
+ nameFormatRow = "ML%03d_NX%03d_KX%02d"
+ nameFormatCol = "MX%03d_NL%03d_KX%02d"
+ nameFormatCorner = "ML%03d_NL%03d_KX%02d"
+
+ ##############################################################################
+ # Tile - constructors
+ ##############################################################################
+ def __init__(self):
+ self.workGroupNumRows = -1
+ self.workGroupNumCols = -1
+ self.microTileNumRows = -1
+ self.microTileNumCols = -1
+ self.macroTileNumRows = -1
+ self.macroTileNumCols = -1
+ self.unroll = -1
+
+ def __eq__(self, other):
+ return self.workGroupNumRows == other.workGroupNumRows \
+ and self.workGroupNumCols == other.workGroupNumCols \
+ and self.microTileNumRows == other.microTileNumRows \
+ and self.microTileNumCols == other.microTileNumCols \
+ and self.unroll == other.unroll
+ def __ni__(self, other):
+ return not self.__eq__(other)
+ def __hash__(self):
+ return \
+ self.workGroupNumRows*2*8*8*256 + \
+ self.workGroupNumCols*2*8*8 + \
+ self.microTileNumRows*2*8 + \
+ self.microTileNumCols*2 + \
+ self.unroll
+ def __str__(self):
+ return self.getName()
+ def __repr__(self):
+ return self.getName()
+
+
+ def printAttributes(self):
+ print "workGroupNumRows = %d" % self.workGroupNumRows
+ print "workGroupNumCols = %d" % self.workGroupNumCols
+ print "microTileNumRows = %d" % self.microTileNumRows
+ print "microTileNumCols = %d" % self.microTileNumCols
+ print "macroTileNumRows = %d" % self.macroTileNumRows
+ print "macroTileNumCols = %d" % self.macroTileNumCols
+ print "unroll = %d" % self.unroll
+
+ ##############################################################################
+ # Tile - get Multiples
+ ##############################################################################
+ def getMultipleM(self):
+ return (self.workGroupNumRows * self.microTileNumRows)
+ def getMultipleN(self):
+ return (self.workGroupNumCols * self.microTileNumCols)
+ def getMultipleK(self):
+ return (self.unroll)
+
+
+ ##############################################################################
+ # Tile - are tile parameters valid?
+ ##############################################################################
+ def isValid(self):
+ return True
+ """
+ numALoads = (self.workGroupNumRows*self.microTileNumRows*self.unroll) \
+ / (self.workGroupNumRows*self.workGroupNumCols)
+ numALoadsR = (self.workGroupNumRows*self.microTileNumRows*self.unroll) \
+ % (self.workGroupNumRows*self.workGroupNumCols)
+ numBLoads = (self.workGroupNumCols*self.microTileNumCols*self.unroll) \
+ / (self.workGroupNumRows*self.workGroupNumCols)
+ numBLoadsR = (self.workGroupNumCols*self.microTileNumCols*self.unroll) \
+ % (self.workGroupNumRows*self.workGroupNumCols)
+ if (numALoads>0 and numALoadsR>0):
+ self.error = ("(%2d * %d * %d = %3d) A elements can't be loaded "
+ "by (%2d * %2d = %3d) threads" ) \
+ % ( self.workGroupNumRows, self.microTileNumRows, self.unroll, \
+ (self.workGroupNumRows*self.microTileNumRows*self.unroll), \
+ self.workGroupNumRows, self.workGroupNumCols, \
+ (self.workGroupNumRows*self.workGroupNumCols) )
+ return False
+ elif (numBLoads>0 and numBLoadsR>0):
+ self.error = ( "(%2d * %d * %d = %3d) B elements can't be loaded "
+ "by (%2d * %2d = %3d) threads" ) \
+ % ( self.workGroupNumCols, self.microTileNumCols, self.unroll, \
+ (self.workGroupNumCols*self.microTileNumCols*self.unroll), \
+ self.workGroupNumRows, self.workGroupNumCols, \
+ (self.workGroupNumRows*self.workGroupNumCols) )
+ return False
+ else:
+ return True
+ """
+
+
+ ##############################################################################
+ # Tile - get Name
+ ##############################################################################
+ def getName(self):
+ if self.macroTileNumRows < self.workGroupNumRows*self.microTileNumRows:
+ if self.macroTileNumCols < self.workGroupNumCols*self.microTileNumCols:
+ return self.nameFormatCorner \
+ % ( (self.workGroupNumRows*self.microTileNumRows), \
+ (self.workGroupNumCols*self.microTileNumCols), self.unroll )
+ else:
+ return self.nameFormatRow \
+ % ( (self.workGroupNumRows*self.microTileNumRows), \
+ (self.workGroupNumCols*self.microTileNumCols), self.unroll )
+ else:
+ if self.macroTileNumCols < self.workGroupNumCols*self.microTileNumCols:
+ return self.nameFormatCol \
+ % ( (self.workGroupNumRows*self.microTileNumRows), \
+ (self.workGroupNumCols*self.microTileNumCols), self.unroll )
+ else:
+ return self.nameFormatTile \
+ % ( (self.workGroupNumRows*self.microTileNumRows), \
+ (self.workGroupNumCols*self.microTileNumCols), self.unroll )
+ def getRowName(self):
+ return self.nameFormatRow \
+ % ( (self.workGroupNumRows*self.microTileNumRows), \
+ (self.workGroupNumCols*self.microTileNumCols), self.unroll )
+ def getColName(self):
+ return self.nameFormatCol \
+ % ( (self.workGroupNumRows*self.microTileNumRows), \
+ (self.workGroupNumCols*self.microTileNumCols), self.unroll )
+ def getCornerName(self):
+ return self.nameFormatCorner \
+ % ( (self.workGroupNumRows*self.microTileNumRows), \
+ (self.workGroupNumCols*self.microTileNumCols), self.unroll )
+
+ ##############################################################################
+ # Row Kernel
+ # - macroTileNumRows = 1
+ # - guards around gA -> lA
+ # - guards around gC[gRow,:] = rC[row,:]
+ ##############################################################################
+ def isRowKernel(self):
+ if self.workGroupNumRows * self.microTileNumRows == self.macroTileNumRows:
+ return False; # normal tile kernel
+ else:
+ if self.macroTileNumRows == 1:
+ return True; # single row kernel
+ else:
+ printf( ("ERROR: workGroupNumRows=%u, microTileNumRows=%u "
+ "and macroTileNumRows=%u doesn't make sense\n") \
+ % (self.workGroupNumRows, self.microTileNumRows, \
+ self.macroTileNumRows) );
+ return False; # ERROR
+
+ ##############################################################################
+ # Col Kernel
+ # - macroTileNumCols = 1
+ # - guards around gB -> lB
+ # - guards around gC[:,gCol] = rC[:,col]
+ ##############################################################################
+ def isColKernel(self):
+ if self.workGroupNumCols * self.microTileNumCols == self.macroTileNumCols:
+ return False; # normal tile kernel
+ else:
+ if self.macroTileNumCols == 1:
+ return True; # single row kernel
+ else:
+ printf(("ERROR: workGroupNumCols=%u, microTileNumCols=%u "
+ "and macroTileNumCols=%u doesn't make sense\n") \
+ % (self.workGroupNumCols, self.microTileNumCols, \
+ self.macroTileNumCols) );
+ return False; # ERROR
+
+
+
+################################################################################
+# Non Tile Parameters
+# - parameters which must match matrix system for correct answer
+################################################################################
+class NonTileParameters:
+ def __init__(self):
+ self.precision = "" # s, d, c, z
+ self.order = "" # clblasColumnMajor, clblasRowMajor
+ self.transA = "" # N, T, C
+ self.transB = "" # N, T, C
+ self.beta = -1 # 0, 1
+
+ def printAttributes(self):
+ print "precision = " + self.precision
+ print "order = " + self.order
+ print "transA = " + self.transA
+ print "transB = " + self.transB
+ print "beta = %d" % self.beta
+
+ ##############################################################################
+ # NonTile - get Name
+ ##############################################################################
+ def getName(self):
+ return "%sgemm_%3s_%1s%1s_B%d" \
+ % (Common.hostDataChar[self.precision], \
+ "Col" if self.order=="clblasColumnMajor" else "Row", \
+ self.transA, self.transB, self.beta )
+
+
+
+################################################################################
+# Kernel Parameters
+################################################################################
+class KernelParameters( NonTileParameters, TileParameters ):
+
+ ##############################################################################
+ # Kernel - constructor
+ ##############################################################################
+ def __init__(self):
+ NonTileParameters.__init__(self)
+ TileParameters.__init__(self)
+ self.localRowPad = 0
+ self.localColPad = 0
+
+ ##############################################################################
+ # Kernel - use tile
+ ##############################################################################
+ def useTile(self, tile):
+ self.workGroupNumRows = tile.workGroupNumRows
+ self.workGroupNumCols = tile.workGroupNumCols
+ self.microTileNumRows = tile.microTileNumRows
+ self.microTileNumCols = tile.microTileNumCols
+ self.macroTileNumRows = tile.macroTileNumRows
+ self.macroTileNumCols = tile.macroTileNumCols
+ self.unroll = tile.unroll
+
+ def printAttributes(self):
+ NonTileParameters.printAttributes(self)
+ TileParameters.printAttributes(self)
+
+ ##############################################################################
+ # Kernel - get Name
+ ##############################################################################
+ def getName(self):
+ return NonTileParameters.getName(self) \
+ + "_" + TileParameters.getName(self)
+ def getRowName(self):
+ return NonTileParameters.getName(self) \
+ + "_" + TileParameters.getRowName(self)
+ def getColName(self):
+ return NonTileParameters.getName(self) \
+ + "_" + TileParameters.getColName(self)
+ def getCornerName(self):
+ return NonTileParameters.getName(self) \
+ + "_" + TileParameters.getCornerName(self)
+
diff --git a/src/library/blas/AutoGemm/KernelSelection.py b/src/library/blas/AutoGemm/KernelSelection.py
new file mode 100644
index 0000000..2c05771
--- /dev/null
+++ b/src/library/blas/AutoGemm/KernelSelection.py
@@ -0,0 +1,683 @@
+import os
+import sys
+import copy
+
+import AutoGemmParameters
+import Common
+import KernelParameters
+
+def indent(il):
+ returnTabs = ""
+ for i in range(0, il):
+ returnTabs += " "
+ return returnTabs
+
+def tileInRange( tileMin, tileMax, rangeMin, rangeMax):
+ if ( tileMax < 0 or (tileMax >= rangeMax and rangeMax>0) ) and tileMin <= rangeMin :
+ valid = True
+ else:
+ valid = False
+ #print "Range [%4ux%4u]: [%4u,%4u] is %s b/c" \
+ # % (rangeMin, rangeMax, tileMin, tileMax, "valid" if valid else "INVALID" )
+ #print "if ( %i<0 or (%u >= %u and %u>0) and %u <= %u" \
+ # %( tileMax, tileMax, rangeMax, rangeMax, tileMin, rangeMin )
+ return valid
+
+
+################################################################################
+# KSL - Kernel Selection Logic File
+################################################################################
+class KernelSelection:
+
+
+
+ ##############################################################################
+ # KSL - default constructor
+ ##############################################################################
+ def __init__( \
+ self, \
+ precisionList, \
+ orderList, \
+ transDict, \
+ betaList, \
+ unrollDict, \
+ kernelSelectionData):
+
+ self.incFileName = Common.getIncludePath() + "AutoGemmKernelSelection.h"
+ self.incFile = open(self.incFileName, "w")
+ self.incFile.write( Common.getAutoGemmHeader() )
+
+ self.kernelSelectionFileName = Common.getIncludePath() + "AutoGemmKernelSelection.cpp"
+ self.selectionFile = open(self.kernelSelectionFileName, "w")
+ self.selectionFile.write( Common.getAutoGemmHeader() )
+
+
+ self.inc = (
+ "#include <clBLAS.h>\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelSources.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBinaries.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBuildOptionsSource.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBuildOptionsBinary.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmClKernels.h\"\n"
+ "\n"
+ "#define EXACT_MULTIPLES(MULTIPLE_STR) MULTIPLE_STR\n"
+ "\n"
+ "// kernel selection logic template\n"
+ "template<typename Precision>\n"
+ "void gemmSelectKernel(\n"
+ " clblasOrder order,\n"
+ " clblasTranspose transA,\n"
+ " clblasTranspose transB,\n"
+ " size_t M,\n"
+ " size_t N,\n"
+ " size_t K,\n"
+ " bool betaNonZero,\n"
+ " float optimalNumElementsPerWorkItem,\n"
+ " const char **tileKernelSource,\n"
+ " const char **rowKernelSource,\n"
+ " const char **colKernelSource,\n"
+ " const char **cornerKernelSource,\n"
+ " const char **sourceBuildOptions,\n"
+ " const unsigned char **tileKernelBinary,\n"
+ " const unsigned char **rowKernelBinary,\n"
+ " const unsigned char **colKernelBinary,\n"
+ " const unsigned char **cornerKernelBinary,\n"
+ " size_t **tileKernelBinarySize,\n"
+ " size_t **rowKernelBinarySize,\n"
+ " size_t **colKernelBinarySize,\n"
+ " size_t **cornerKernelBinarySize,\n"
+ " const char **binaryBuildOptions,\n"
+ " cl_kernel **tileClKernel,\n"
+ " cl_kernel **rowClKernel,\n"
+ " cl_kernel **colClKernel,\n"
+ " cl_kernel **cornerClKernel,\n"
+ " unsigned int *workGroupNumRows,\n"
+ " unsigned int *workGroupNumCols,\n"
+ " unsigned int *microTileNumRows,\n"
+ " unsigned int *microTileNumCols,\n"
+ " unsigned int *unroll\n"
+ ");\n\n" )
+
+ self.logic = "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelSelection.h\"\n"
+
+ ####################################
+ # precision
+ kernel = KernelParameters.KernelParameters()
+ for precision in precisionList:
+ #self.selectionFile.write( self.logic )
+ #self.logic = ""
+ kernel.precision = precision
+ sizeEvents = kernelSelectionData[precision]
+ self.logic += (
+ "\n// " + precision + "gemm kernel selection logic\n"
+ "template<>\n"
+ "void gemmSelectKernel<" )
+ if precision == "s":
+ self.logic += "float"
+ elif precision == "d":
+ self.logic += "double"
+ elif precision == "c":
+ self.logic += "FloatComplex"
+ else:
+ self.logic += "DoubleComplex"
+
+ self.logic += (
+ ">(\n"
+ " clblasOrder order,\n"
+ " clblasTranspose transA,\n"
+ " clblasTranspose transB,\n"
+ " size_t M,\n"
+ " size_t N,\n"
+ " size_t K,\n"
+ " bool betaNonZero,\n"
+ " float optimalNumElementsPerWorkItem,\n"
+ " const char **tileKernelSource,\n"
+ " const char **rowKernelSource,\n"
+ " const char **colKernelSource,\n"
+ " const char **cornerKernelSource,\n"
+ " const char **sourceBuildOptions,\n"
+ " const unsigned char **tileKernelBinary,\n"
+ " const unsigned char **rowKernelBinary,\n"
+ " const unsigned char **colKernelBinary,\n"
+ " const unsigned char **cornerKernelBinary,\n"
+ " size_t **tileKernelBinarySize,\n"
+ " size_t **rowKernelBinarySize,\n"
+ " size_t **colKernelBinarySize,\n"
+ " size_t **cornerKernelBinarySize,\n"
+ " const char **binaryBuildOptions,\n"
+ " cl_kernel **tileClKernel,\n"
+ " cl_kernel **rowClKernel,\n"
+ " cl_kernel **colClKernel,\n"
+ " cl_kernel **cornerClKernel,\n"
+ " unsigned int *workGroupNumRows,\n"
+ " unsigned int *workGroupNumCols,\n"
+ " unsigned int *microTileNumRows,\n"
+ " unsigned int *microTileNumCols,\n"
+ " unsigned int *unroll\n"
+ ") {\n" )
+
+ ####################################
+ # order
+ for order in orderList:
+ #print precision + "gemm" + "_" + order
+ kernel.order = order
+ self.logic += indent(1) + "if (order == " + order + ") {\n"
+ transList = transDict[precision]
+
+ ####################################
+ # transA
+ for transA in transList:
+ #print precision + "gemm" + "_" + order + "_" + transA
+ kernel.transA = transA
+ self.logic += indent(2) + "if (transA == "
+ if transA == "N":
+ self.logic += "clblasNoTrans"
+ elif transA == "T":
+ self.logic += "clblasTrans"
+ else:
+ self.logic += "clblasConjTrans"
+ self.logic += ") {\n"
+
+ ####################################
+ # transB
+ for transB in transList:
+ kernel.transB = transB
+ self.logic += indent(3) + "if (transB == "
+ if transB == "N":
+ self.logic += "clblasNoTrans"
+ elif transB == "T":
+ self.logic += "clblasTrans"
+ else:
+ self.logic += "clblasConjTrans"
+ self.logic += ") {\n"
+
+ ####################################
+ # beta
+ for beta in betaList:
+ #print precision + "gemm" + "_" + order + "_" + transA + "_" + transB + "_B" + str(beta)
+ kernel.beta = beta
+ self.logic += indent(4) + "if ( "
+ if beta == 0:
+ self.logic += "!betaNonZero"
+ else:
+ self.logic += "betaNonZero"
+ self.logic += " ) {\n"
+
+ ####################################
+ # if size event
+ for sizeEvent in sizeEvents:
+ self.selectionFile.write( self.logic )
+ self.logic = ""
+ sizeMin = sizeEvent[0]
+ fallbackTile = sizeEvent[1]
+ validTiles = sizeEvent[2]
+ self.logic += indent(5)+"if ( M*N >= "+str(sizeMin)+"*"+str(sizeMin) + ") {\n"
+ #print precision + "gemm" + "_" + order + "_" + transA + "_" + transB + "_B" + str(beta) + "_" + str(sizeMin) + "->" + str(sizeMax)
+
+ ####################################
+ # valid tiles
+ self.logic += indent(6)+"// valid tiles\n"
+ for tileParams in validTiles:
+ kernel.workGroupNumRows = tileParams[0]
+ kernel.workGroupNumCols = tileParams[1]
+ kernel.microTileNumRows = tileParams[2]
+ kernel.microTileNumCols = tileParams[3]
+ kernel.macroTileNumRows = kernel.workGroupNumRows*kernel.microTileNumRows
+ kernel.macroTileNumCols = kernel.workGroupNumCols*kernel.microTileNumCols
+ for unroll in unrollDict[precision]:
+ kernel.unroll = unroll
+ self.logic += indent(6)+"if ( M%%%d == 0 && N%%%d == 0 && K%%%d == 0) {\n" \
+ % (kernel.getMultipleM(), kernel.getMultipleN(), kernel.getMultipleK())
+ self.addBodyForKernel( kernel )
+ self.logic += indent(6) + "}\n"
+
+ ####################################
+ # fallback tile - TODO all tiles begin added
+ self.logic += indent(6)+"// fallback tile\n"
+ #print "\nFallback[%i, %i]"%(sizeMin, sizeMax)
+ kernel.workGroupNumRows = fallbackTile[0]
+ kernel.workGroupNumCols = fallbackTile[1]
+ kernel.microTileNumRows = fallbackTile[2]
+ kernel.microTileNumCols = fallbackTile[3]
+ kernel.macroTileNumRows = kernel.workGroupNumRows*kernel.microTileNumRows
+ kernel.macroTileNumCols = kernel.workGroupNumCols*kernel.microTileNumCols
+ for unroll in unrollDict[precision]:
+ kernel.unroll = unroll
+ self.logic += indent(6)+"if ( K%%%d == 0 ) {\n" \
+ % (kernel.getMultipleK())
+ self.addBodyForKernel( kernel )
+ self.logic += indent(6) + "}\n"
+
+ ####################################
+ # end size event
+ self.logic += indent(5) + "} // end size\n"
+
+ ####################################
+ # end beta
+ self.logic += indent(4) + "} // end beta\n"
+
+ ####################################
+ # end transB
+ self.logic += indent(3) + "} // end transB\n"
+
+ ####################################
+ # end transA
+ self.logic += indent(2) + "} // end transA\n"
+
+ ####################################
+ # end order
+ self.logic += indent(1) + "} // end order\n"
+
+ ####################################
+ # end precision
+ self.logic += indent(0) + "} // end precision function\n"
+ # write last precision
+ self.selectionFile.write( self.logic )
+ self.selectionFile.write( "\n" )
+
+
+
+ def addBodyForKernel( self, kernel ):
+ #self.logic += indent(7) + "printf(\"selected kernel: " + kernel.getName() + "\\n\");\n"
+ self.logic += indent(7) + "*tileKernelSource = " + kernel.getName() + "_src;\n"
+ self.logic += indent(7) + "*rowKernelSource = " + kernel.getRowName() + "_src;\n"
+ self.logic += indent(7) + "*colKernelSource = " + kernel.getColName() + "_src;\n"
+ self.logic += indent(7) + "*cornerKernelSource = " + kernel.getCornerName() + "_src;\n"
+ self.logic += indent(7) + "*sourceBuildOptions = " + kernel.getName() + "_srcBuildOptions;\n"
+ self.logic += indent(7) + "*tileKernelBinary = " + kernel.getName() + "_bin;\n"
+ self.logic += indent(7) + "*rowKernelBinary = " + kernel.getRowName() + "_bin;\n"
+ self.logic += indent(7) + "*colKernelBinary = " + kernel.getColName() + "_bin;\n"
+ self.logic += indent(7) + "*cornerKernelBinary = " + kernel.getCornerName() + "_bin;\n"
+ self.logic += indent(7) + "*tileKernelBinarySize = &" + kernel.getName() + "_binSize;\n"
+ self.logic += indent(7) + "*rowKernelBinarySize = &" + kernel.getRowName() + "_binSize;\n"
+ self.logic += indent(7) + "*colKernelBinarySize = &" + kernel.getColName() + "_binSize;\n"
+ self.logic += indent(7) + "*cornerKernelBinarySize = &" + kernel.getCornerName() + "_binSize;\n"
+ self.logic += indent(7) + "*binaryBuildOptions = " + kernel.getName() + "_binBuildOptions;\n"
+ self.logic += indent(7) + "*tileClKernel = &" + kernel.getName() + "_clKernel;\n"
+ self.logic += indent(7) + "*rowClKernel = &" + kernel.getRowName() + "_clKernel;\n"
+ self.logic += indent(7) + "*colClKernel = &" + kernel.getColName() + "_clKernel;\n"
+ self.logic += indent(7) + "*cornerClKernel = &" + kernel.getCornerName() + "_clKernel;\n"
+ self.logic += indent(7) + "*workGroupNumRows = " + kernel.getName() + "_workGroupNumRows;\n"
+ self.logic += indent(7) + "*workGroupNumCols = " + kernel.getName() + "_workGroupNumCols;\n"
+ self.logic += indent(7) + "*microTileNumRows = " + kernel.getName() + "_microTileNumRows;\n"
+ self.logic += indent(7) + "*microTileNumCols = " + kernel.getName() + "_microTileNumRows;\n"
+ self.logic += indent(7) + "*unroll = " + kernel.getName() + "_unroll;\n"
+ self.logic += indent(7) + "return;\n"
+
+
+
+ ##############################################################################
+ # KSL - write to file
+ ##############################################################################
+ def writeToFile(self):
+ self.selectionFile.close()
+
+ self.incFile.write( self.inc )
+ self.incFile.close()
+
+
+
+
+################################################################################
+# KSM - Kernel Selection Manual/Specific File
+################################################################################
+class KernelSelectionSpecific:
+
+ zeroIndent = " "
+ tab = " "
+ ##############################################################################
+ # KSL - default constructor
+ ##############################################################################
+ def __init__(self):
+
+ self.incFileName = Common.getIncludePath() + "AutoGemmKernelSelectionSpecific.h"
+ self.incFile = open(self.incFileName, "w")
+ self.incFile.write( Common.getAutoGemmHeader() )
+
+ self.kernelSelectionFileName = Common.getIncludePath() + "AutoGemmKernelSelectionSpecific.cpp"
+ self.selectionFile = open(self.kernelSelectionFileName, "w")
+ self.selectionFile.write( Common.getAutoGemmHeader() )
+
+ self.inc = (
+ "#include <clBLAS.h>\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelSources.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBinaries.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBuildOptionsSource.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelBuildOptionsBinary.h\"\n"
+ "#include \"" + Common.getRelativeIncludePath() + "AutoGemmClKernels.h\"\n"
+ "\n"
+ "// kernel selection specific template\n"
+ "template<typename Precision>\n"
+ "bool gemmSelectKernelSpecific(\n"
+ " clblasOrder order,\n"
+ " clblasTranspose transA,\n"
+ " clblasTranspose transB,\n"
+ " bool betaNonZero,\n"
+ " unsigned int macroTileNumRows,\n"
+ " unsigned int macroTileNumCols,\n"
+ " unsigned int unroll,\n"
+ " const char **tileKernelSource,\n"
+ " const char **rowKernelSource,\n"
+ " const char **colKernelSource,\n"
+ " const char **cornerKernelSource,\n"
+ " const char **sourceBuildOptions,\n"
+ " const unsigned char **tileKernelBinary,\n"
+ " const unsigned char **rowKernelBinary,\n"
+ " const unsigned char **colKernelBinary,\n"
+ " const unsigned char **cornerKernelBinary,\n"
+ " size_t **tileKernelBinarySize,\n"
+ " size_t **rowKernelBinarySize,\n"
+ " size_t **colKernelBinarySize,\n"
+ " size_t **cornerKernelBinarySize,\n"
+ " const char **binaryBuildOptions,\n"
+ " cl_kernel **tileClKernel,\n"
+ " cl_kernel **rowClKernel,\n"
+ " cl_kernel **colClKernel,\n"
+ " cl_kernel **cornerClKernel,\n"
+ " unsigned int *workGroupNumRows,\n"
+ " unsigned int *workGroupNumCols,\n"
+ " unsigned int *microTileNumRows,\n"
+ " unsigned int *microTileNumCols\n"
+ ");\n\n" )
+
+ self.logic = "#include \"" + Common.getRelativeIncludePath() + "AutoGemmKernelSelectionSpecific.h\"\n"
+ self.precisionInitialized = False
+ self.orderInitialized = False
+ self.transInitialized = False
+ self.betaInitialized = False
+
+ def newPrecision(self, precision ):
+ #print "KernelSelectionSpecific: " + precision + "gemm"
+ if self.precisionInitialized:
+ self.logic += self.zeroIndent+self.tab+self.tab + "}\n" # 2 tabs
+ self.logic += self.zeroIndent+self.tab + "}\n" # 1 tab
+ self.logic += self.zeroIndent+"}\n"
+ self.logic += self.zeroIndent + "return false; // didn't find a match\n"
+ self.logic += "}\n\n"
+ else:
+ self.logic += self.zeroIndent
+
+ self.logic += (
+ "\n// " + precision + "gemm kernel selection specific\n"
+ "template<>\n"
+ "bool gemmSelectKernelSpecific<" )
+ if precision == "s":
+ self.logic += "float"
+ elif precision == "d":
+ self.logic += "double"
+ elif precision == "c":
+ self.logic += "FloatComplex"
+ else:
+ self.logic += "DoubleComplex"
+
+ self.logic += (
+ ">(\n"
+ " clblasOrder order,\n"
+ " clblasTranspose transA,\n"
+ " clblasTranspose transB,\n"
+ " bool betaNonZero,\n"
+ " unsigned int macroTileNumRows,\n"
+ " unsigned int macroTileNumCols,\n"
+ " unsigned int unroll,\n"
+ " const char **tileKernelSource,\n"
+ " const char **rowKernelSource,\n"
+ " const char **colKernelSource,\n"
+ " const char **cornerKernelSource,\n"
+ " const char **sourceBuildOptions,\n"
+ " const unsigned char **tileKernelBinary,\n"
+ " const unsigned char **rowKernelBinary,\n"
+ " const unsigned char **colKernelBinary,\n"
+ " const unsigned char **cornerKernelBinary,\n"
+ " size_t **tileKernelBinarySize,\n"
+ " size_t **rowKernelBinarySize,\n"
+ " size_t **colKernelBinarySize,\n"
+ " size_t **cornerKernelBinarySize,\n"
+ " const char **binaryBuildOptions,\n"
+ " cl_kernel **tileClKernel,\n"
+ " cl_kernel **rowClKernel,\n"
+ " cl_kernel **colClKernel,\n"
+ " cl_kernel **cornerClKernel,\n"
+ " unsigned int *workGroupNumRows,\n"
+ " unsigned int *workGroupNumCols,\n"
+ " unsigned int *microTileNumRows,\n"
+ " unsigned int *microTileNumCols\n"
+ ") {\n" )
+ self.precisionInitialized = True
+ self.orderInitialized = False
+ self.transInitialized = False
+ self.betaInitialized = False
+
+
+ ####################################
+ # KSL - new order
+ def newOrder(self, order):
+ if (self.orderInitialized):
+ self.logic += self.zeroIndent+self.tab+self.tab + "}\n" # 2 tabs
+ self.logic += self.zeroIndent+self.tab + "}\n" # 1 tab
+ self.logic += self.zeroIndent
+ self.logic += "} else "
+ else:
+ self.logic += self.zeroIndent
+ self.logic += "if (order == " + order + ") {\n"
+ self.orderInitialized = True
+ self.transInitialized = False
+ self.betaInitialized = False
+
+
+ ####################################
+ # KSL - new trans
+ def newTrans(self, transA, transB):
+ if (self.transInitialized):
+ self.logic += self.zeroIndent+self.tab+self.tab + "}\n" # 2 tabs
+ self.logic += self.zeroIndent+self.tab # 1 tab
+ self.logic += "} else "
+ else:
+ self.logic += self.zeroIndent+self.tab # 1 tabs
+ self.logic += "if (transA == "
+ if transA == "N":
+ self.logic += "clblasNoTrans"
+ elif transA == "T":
+ self.logic += "clblasTrans"
+ else:
+ self.logic += "clblasConjTrans"
+ self.logic += " && transB == "
+ if transB == "N":
+ self.logic += "clblasNoTrans"
+ elif transB == "T":
+ self.logic += "clblasTrans"
+ else:
+ self.logic += "clblasConjTrans"
+ self.logic += ") {\n"
+ self.transInitialized = True
+ self.betaInitialized = False
+
+ ####################################
+ # KSL - new beta
+ def newBeta(self, beta):
+ if (self.betaInitialized):
+ self.logic += self.zeroIndent+self.tab+self.tab # 2 tabs
+ self.logic += "} else "
+ else:
+ self.logic += self.zeroIndent+self.tab+self.tab # 2 tabs
+ self.logic += "if ( "
+ if beta == 0:
+ self.logic += "!betaNonZero"
+ else:
+ self.logic += "betaNonZero"
+ self.logic += " ) {\n"
+ self.betaInitialized = True
+
+
+ ##############################################################################
+ # KSL - add new kernel
+ ##############################################################################
+ def newKernel(self, kernel):
+
+ # new kernel
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab # 3 tabs
+ self.logic += ("if ( macroTileNumRows == %u && macroTileNumCols == %u "
+ "&& unroll == %u) {\n") \
+ % ( kernel.macroTileNumRows, kernel.macroTileNumCols, kernel.unroll )
+
+ #self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab+self.tab # 5 tabs
+ #self.logic += "printf(\"selected kernel: " + kernel.getName() + "\\n\");\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*tileKernelSource = " + kernel.getName() + "_src;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*rowKernelSource = " + kernel.getRowName() + "_src;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*colKernelSource = " + kernel.getColName() + "_src;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*cornerKernelSource = " + kernel.getCornerName() + "_src;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*sourceBuildOptions = " + kernel.getName() + "_srcBuildOptions;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*tileKernelBinary = " + kernel.getName() + "_bin;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*rowKernelBinary = " + kernel.getRowName() + "_bin;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*colKernelBinary = " + kernel.getColName() + "_bin;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*cornerKernelBinary = " + kernel.getCornerName() + "_bin;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*tileKernelBinarySize = &" + kernel.getName() + "_binSize;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*rowKernelBinarySize = &" + kernel.getRowName() + "_binSize;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*colKernelBinarySize = &" + kernel.getColName() + "_binSize;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*cornerKernelBinarySize = &" + kernel.getCornerName() + "_binSize;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*binaryBuildOptions = " + kernel.getName() + "_binBuildOptions;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*tileClKernel = &" + kernel.getName() + "_clKernel;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*rowClKernel = &" + kernel.getRowName() + "_clKernel;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*colClKernel = &" + kernel.getColName() + "_clKernel;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*cornerClKernel = &" + kernel.getCornerName() + "_clKernel;\n"
+ # dims
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*workGroupNumRows = " + kernel.getName() + "_workGroupNumRows;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*workGroupNumCols = " + kernel.getName() + "_workGroupNumCols;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*microTileNumRows = " + kernel.getName() + "_microTileNumRows;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "*microTileNumCols = " + kernel.getName() + "_microTileNumCols;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab+self.tab # 4 tabs
+ self.logic += "return true;\n"
+
+ self.logic += self.zeroIndent+self.tab+self.tab+self.tab # 3 tabs
+ self.logic += "}\n"
+
+ self.selectionFile.write( self.logic )
+ self.logic = ""
+
+
+ ##############################################################################
+ # KSL - write to file
+ ##############################################################################
+ def writeToFile(self):
+ self.logic += self.zeroIndent+self.tab+self.tab + "}\n" # 2 tabs
+ self.logic += self.zeroIndent+self.tab + "}\n" # 1 tab
+ self.logic += self.zeroIndent + "}\n" # 0 tab
+ self.logic += self.zeroIndent + "return false; // didn't find a match\n"
+ self.logic += "}\n" # close function
+
+ self.selectionFile.write(self.logic)
+ self.selectionFile.write("\n")
+ self.selectionFile.close()
+
+ self.incFile.write(self.inc)
+ self.incFile.write("\n")
+ self.incFile.close()
+
+
+################################################################################
+# Main
+################################################################################
+def writeKernelSelection():
+ print "AutoGemm.py: Generating kernel selection."
+ if not os.path.exists( Common.getIncludePath() ):
+ os.makedirs( Common.getIncludePath() )
+
+ ########################################
+ # kernel selection specific
+ kss = KernelSelectionSpecific()
+
+ # for each precision
+ kernel = KernelParameters.KernelParameters()
+ for precision in AutoGemmParameters.precisions:
+ kernel.precision = precision
+ kss.newPrecision(precision)
+
+ # valid tiles for this precision
+ tiles = AutoGemmParameters.getTilesForPrecision(precision)
+
+ # for non tile parameters
+ for order in AutoGemmParameters.orders:
+ kernel.order = order
+ kss.newOrder(order)
+ for transA in AutoGemmParameters.transposes[precision]:
+ kernel.transA = transA
+ for transB in AutoGemmParameters.transposes[precision]:
+ kernel.transB = transB
+ kss.newTrans(transA, transB)
+ for beta in AutoGemmParameters.betas:
+ kernel.beta = beta
+ kss.newBeta(beta)
+
+ # for tile parameters
+ for tile in tiles:
+ kernel.useTile(tile)
+ kss.newKernel(kernel)
+
+ kss.writeToFile()
+
+ ########################################
+ # kernel selection
+ ks = KernelSelection( \
+ AutoGemmParameters.precisions, \
+ AutoGemmParameters.orders, \
+ AutoGemmParameters.transposes, \
+ AutoGemmParameters.betas, \
+ AutoGemmParameters.unrolls, \
+ AutoGemmParameters.kernelSelectionData )
+ ks.writeToFile()
+
+
+
+################################################################################
+# Main
+################################################################################
+if __name__ == "__main__":
+ if len(sys.argv) == 2:
+ Common.setOutputPath(sys.argv[1])
+ else:
+ print "Warning: No output path specified; default is working directory."
+ writeKernelSelection()
+
diff --git a/src/library/blas/AutoGemm/KernelsToPreCompile.py b/src/library/blas/AutoGemm/KernelsToPreCompile.py
new file mode 100644
index 0000000..ff24c50
--- /dev/null
+++ b/src/library/blas/AutoGemm/KernelsToPreCompile.py
@@ -0,0 +1,91 @@
+import os
+import argparse
+import AutoGemmParameters
+import Common
+
+
+################################################################################
+# Auto-Gemm
+################################################################################
+
+def writeOfflineCompilation(args):
+ print "AutoGemm.py: Generating list of kernels to pre-compile."
+ if not os.path.exists( Common.getIncludePath() ):
+ os.makedirs( Common.getIncludePath() )
+
+ ocFileName = Common.getIncludePath() + "AutoGemmKernelsToPreCompile.h"
+ ocFile = open(ocFileName, "w")
+ ocFile.write( Common.getAutoGemmHeader() )
+
+ fileStr = "\n/*precision, order, transA, transB, beta, tileNumRows, tileNumCols, unroll*/\n"
+ fileStr += "\nunsigned int gemmPreCompile[][8] = {\n"
+
+ count = 0
+ for precision in args.precisions:
+ ocFile.write( fileStr )
+ fileStr = ""
+ validTiles = AutoGemmParameters.getTilesForPrecision(precision)
+ for order in args.orders:
+ for transpose in args.transposes:
+ transA = transpose[0]
+ transB = transpose[1]
+ if (transA=="C" or transB=="C") and (precision=="s" or precision=="d"):
+ # real precision doesn't have conjugate transpose
+ continue
+ for beta in args.betas:
+ for tile in validTiles:
+ # print combination
+ kernelStr = " { %1u, %1u, %1u, %1u, %1u, %3u, %3u, %2u },\n" \
+ % (
+ Common.precisionInt[precision],
+ Common.orderInt[order],
+ Common.transposeInt[transA],
+ Common.transposeInt[transB],
+ beta,
+ tile.macroTileNumRows,
+ tile.macroTileNumCols,
+ tile.unroll
+ )
+ fileStr += kernelStr
+ #print kernelStr
+ count+=1
+ if count is 0:
+ fileStr += " { %1u, %1u, %1u, %1u, %1u, %3u, %3u, %2u },\n" \
+ % ( 0, 0, 0, 0, 0, 0, 0, 0 )
+ fileStr += "};\n"
+ fileStr += "unsigned int gemmPreCompileNum = " + str(count) + ";\n"
+ ocFile.write( fileStr )
+ ocFile.close()
+ count *= 4
+ print "AutoGemm.py: %u kernels will be pre-compiled." % count
+
+
+################################################################################
+# Main
+################################################################################
+if __name__ == "__main__":
+
+ # parse arguments
+ ap = argparse.ArgumentParser(description="Which gemm kernels to compile offline.")
+ ap.add_argument("--output-path", dest="output" )
+ ap.add_argument("--precisions", dest="precisions", action="store", nargs="+", choices=AutoGemmParameters.precisions )
+ ap.add_argument("--orders", dest="orders", action="store", nargs="+", choices=AutoGemmParameters.orders )
+ ap.add_argument("--transposes", dest="transposes", action="store", nargs="+", choices=AutoGemmParameters.getTransposeChoices() )
+ ap.add_argument("--betas", dest="betas", action="store", nargs="+", type=int, choices=AutoGemmParameters.betas )
+ args = ap.parse_args()
+ if args.output:
+ Common.setOutputPath(args.output)
+ else:
+ print "Warning: No output path specified; default is working directory."
+
+ # write offline compilation header
+ if args.precisions is None:
+ args.precisions = []
+ if args.transposes is None:
+ args.transposes = []
+ if args.orders is None:
+ args.orders = []
+ if args.betas is None:
+ args.betas = []
+ writeOfflineCompilation(args)
+
diff --git a/src/library/blas/AutoGemm/README.txt b/src/library/blas/AutoGemm/README.txt
new file mode 100644
index 0000000..e69de29
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmClKernels.h b/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmClKernels.h
new file mode 100644
index 0000000..908bcf0
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmClKernels.h
@@ -0,0 +1,18 @@
+
+#ifndef USERGEMM_CL_KERNELS_H
+#define USERGEMM_CL_KERNELS_H
+#include "CL/cl.h"
+
+static cl_kernel sgemm_Col_NT_B1_MX128_NX128_KX16_clKernel = NULL;
+
+static cl_kernel sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_clKernel = NULL;
+static cl_kernel sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_clKernel = NULL;
+static cl_kernel sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_clKernel = NULL;
+
+static cl_kernel sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_clKernel = NULL;
+static cl_kernel sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_clKernel = NULL;
+static cl_kernel sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_clKernel = NULL;
+
+static const int user_kernel_count = 7;
+
+#endif
\ No newline at end of file
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.cpp
new file mode 100644
index 0000000..afd0db2
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.cpp
@@ -0,0 +1,57 @@
+/*******************************************************************************
+ * This file is NOT auto-generated; populate it with hand-written kernels
+ * - David Tanner
+ ******************************************************************************/
+
+
+#ifndef USER_GEMM_SOURCE_INCLUDES_CPP
+#define USER_GEMM_SOURCE_INCLUDES_CPP
+
+//**** Kernels to replace auto-generated versions
+#include "UserGemmKernelSources/sgemm_Col_NN_B0_MX032_NX032_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NN_B0_MX064_NX064_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NN_B0_MX096_NX096_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NN_B1_MX064_NX064_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NN_B1_MX096_NX096_KX16_src.cpp"
+
+#include "UserGemmKernelSources/sgemm_Col_NT_B0_MX032_NX032_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B0_MX064_NX064_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B0_MX096_NX096_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX064_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX096_NX096_KX16_src.cpp"
+
+#include "UserGemmKernelSources/sgemm_Col_TN_B0_MX032_NX032_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_TN_B0_MX064_NX064_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_TN_B0_MX096_NX096_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_TN_B1_MX064_NX064_KX16_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_TN_B1_MX096_NX096_KX16_src.cpp"
+
+#include "UserGemmKernelSources/dgemm_Col_NN_B0_MX048_NX048_KX08_src.cpp"
+#include "UserGemmKernelSources/dgemm_Col_NN_B1_MX048_NX048_KX08_src.cpp"
+#include "UserGemmKernelSources/dgemm_Col_NT_B0_MX048_NX048_KX08_src.cpp"
+#include "UserGemmKernelSources/dgemm_Col_NT_B1_MX048_NX048_KX08_src.cpp"
+#include "UserGemmKernelSources/dgemm_Col_TN_B0_MX048_NX048_KX08_src.cpp"
+#include "UserGemmKernelSources/dgemm_Col_TN_B1_MX048_NX048_KX08_src.cpp"
+
+//**** Special kernels without auto-generated counterparts
+//**** micro tile size 8x8 kernel
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX128_NX128_KX16_src.cpp"
+//**** mod32 but not mod64 kernels
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX032_KX16_COL_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src.cpp"
+//**** branch kernels with 32x32 macro tile size
+#include "UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src.cpp"
+#include "UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src.cpp"
+
+
+//**** compiler flags
+//**** online compilation flags
+//const char * const User_srcBuildOptions = "-cl-std=CL2.0";
+//const char * const User_binBuildOptions = " ";
+
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.h b/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.h
new file mode 100644
index 0000000..3f35061
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/UserGemmKernelSourceIncludes.h
@@ -0,0 +1,80 @@
+
+#ifndef USER_GEMM_SOURCE_INCLUDES_H
+#define USER_GEMM_SOURCE_INCLUDES_H
+#include <cstddef>
+
+//#ifdef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+//#include "AutoGemmKernelBinaries/sgemm_Col_NT_B1_MX128_NX128_KX16_bin.cpp"
+//#endif
+
+//**** compiler flags
+//**** online compilation flags
+const char * const User_srcBuildOptions = "-cl-std=CL2.0";
+const char * const User_binBuildOptions = "-cl-std=CL2.0";
+
+
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_workGroupNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_workGroupNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_microTileNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_microTileNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_unroll;
+extern const char * const sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src;
+extern unsigned char *sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_bin;
+extern size_t sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_binSize;
+
+
+extern const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_workGroupNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_workGroupNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_microTileNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_microTileNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_unroll;
+extern const char * const sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_src;
+extern unsigned char *sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_bin;
+extern size_t sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_binSize;
+
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_workGroupNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_workGroupNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_microTileNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_microTileNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_unroll;
+extern const char * const sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src;
+extern unsigned char *sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_bin;
+extern size_t sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_binSize;
+
+extern const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_workGroupNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_workGroupNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_microTileNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_microTileNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_unroll;
+extern const char * const sgemm_Col_NT_B1_MX128_NX128_KX16_src;
+extern unsigned char *sgemm_Col_NT_B1_MX128_NX128_KX16_bin;
+extern size_t sgemm_Col_NT_B1_MX128_NX128_KX16_binSize;
+
+extern const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows;
+extern const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols;
+extern const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_microTileNumRows;
+extern const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_microTileNumCols;
+extern const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_unroll;
+extern const char * const sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src;
+extern unsigned char *sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_bin;
+extern size_t sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_binSize;
+
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_microTileNumRows;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_microTileNumCols;
+extern const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_unroll;
+extern const char * const sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src;
+extern unsigned char *sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_bin;
+extern size_t sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_binSize;
+
+extern const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows;
+extern const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols;
+extern const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_microTileNumRows;
+extern const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_microTileNumCols;
+extern const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_unroll;
+extern const char * const sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src;
+extern unsigned char *sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_bin;
+extern size_t sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_binSize;
+
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NN_B0_MX048_NX048_KX08_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NN_B0_MX048_NX048_KX08_src.cpp
new file mode 100644
index 0000000..7b0a4d9
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NN_B0_MX048_NX048_KX08_src.cpp
@@ -0,0 +1,203 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_DGEMM_COL_NN_B0_MX048_NX048_KX08_SRC_H
+#define KERNEL_DGEMM_COL_NN_B0_MX048_NX048_KX08_SRC_H
+#pragma message("AutoGemm's dgemm_Col_NN_B0_MX048_NX048_KX08_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int dgemm_Col_NN_B0_MX048_NX048_KX08_workGroupNumRows = 8;
+const unsigned int dgemm_Col_NN_B0_MX048_NX048_KX08_workGroupNumCols = 8;
+const unsigned int dgemm_Col_NN_B0_MX048_NX048_KX08_microTileNumRows = 6;
+const unsigned int dgemm_Col_NN_B0_MX048_NX048_KX08_microTileNumCols = 6;
+const unsigned int dgemm_Col_NN_B0_MX048_NX048_KX08_unroll = 8;
+
+const char * const dgemm_Col_NN_B0_MX048_NX048_KX08_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0] = lA[offA + 0];\
+ rA[1] = lA[offA + 8];\
+ rA[2] = lA[offA + 16];\
+ rA[3] = lA[offA + 24];\
+ rA[4] = lA[offA + 32];\
+ rA[5] = lA[offA + 40];\
+ rB[0] = lB[offB + 0];\
+ rB[1] = lB[offB + 8];\
+ rB[2] = lB[offB + 16];\
+ rB[3] = lB[offB + 24]; \
+ rB[4] = lB[offB + 32]; \
+ rB[5] = lB[offB + 40]; \
+ offA += 49; \
+ offB += 49; \
+ rC[0][0]=mad(rA[0],rB[0],rC[0][0]); \
+ rC[1][0]=mad(rA[1],rB[0],rC[1][0]); \
+ rC[2][0]=mad(rA[2],rB[0],rC[2][0]); \
+ rC[3][0]=mad(rA[3],rB[0],rC[3][0]); \
+ rC[4][0]=mad(rA[4],rB[0],rC[4][0]); \
+ rC[5][0]=mad(rA[5],rB[0],rC[5][0]); \
+ rC[0][1]=mad(rA[0],rB[1],rC[0][1]); \
+ rC[1][1]=mad(rA[1],rB[1],rC[1][1]); \
+ rC[2][1]=mad(rA[2],rB[1],rC[2][1]); \
+ rC[3][1]=mad(rA[3],rB[1],rC[3][1]); \
+ rC[4][1]=mad(rA[4],rB[1],rC[4][1]); \
+ rC[5][1]=mad(rA[5],rB[1],rC[5][1]); \
+ rC[0][2]=mad(rA[0],rB[2],rC[0][2]); \
+ rC[1][2]=mad(rA[1],rB[2],rC[1][2]); \
+ rC[2][2]=mad(rA[2],rB[2],rC[2][2]); \
+ rC[3][2]=mad(rA[3],rB[2],rC[3][2]); \
+ rC[4][2]=mad(rA[4],rB[2],rC[4][2]); \
+ rC[5][2]=mad(rA[5],rB[2],rC[5][2]); \
+ rC[0][3]=mad(rA[0],rB[3],rC[0][3]); \
+ rC[1][3]=mad(rA[1],rB[3],rC[1][3]); \
+ rC[2][3]=mad(rA[2],rB[3],rC[2][3]); \
+ rC[3][3]=mad(rA[3],rB[3],rC[3][3]); \
+ rC[4][3]=mad(rA[4],rB[3],rC[4][3]); \
+ rC[5][3]=mad(rA[5],rB[3],rC[5][3]); \
+ rC[0][4]=mad(rA[0],rB[4],rC[0][4]); \
+ rC[1][4]=mad(rA[1],rB[4],rC[1][4]); \
+ rC[2][4]=mad(rA[2],rB[4],rC[2][4]); \
+ rC[3][4]=mad(rA[3],rB[4],rC[3][4]); \
+ rC[4][4]=mad(rA[4],rB[4],rC[4][4]); \
+ rC[5][4]=mad(rA[5],rB[4],rC[5][4]); \
+ rC[0][5]=mad(rA[0],rB[5],rC[0][5]); \
+ rC[1][5]=mad(rA[1],rB[5],rC[1][5]); \
+ rC[2][5]=mad(rA[2],rB[5],rC[2][5]); \
+ rC[3][5]=mad(rA[3],rB[5],rC[3][5]); \
+ rC[4][5]=mad(rA[4],rB[5],rC[4][5]); \
+ rC[5][5]=mad(rA[5],rB[5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(8,8,1)))
+__kernel void dgemm_Col_NN_B0_MX048_NX048_KX08 (
+ __global double const * restrict A,
+ __global double const * restrict B,
+ __global double * C,
+ double const alpha,
+ double const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+
+
+ A += offsetA;
+ B += offsetB;
+ C += offsetC;
+
+
+ double rC[6][6] = {(double)0};
+ double rA[6];
+ double rB[6];
+ __local double lA[392];
+ __local double lB[392];
+
+ int gidx = get_group_id(0);
+ int gidy = get_group_id(1);
+ int idx = get_local_id(0);
+ int idy = get_local_id(1);
+
+ int idt = 8*idy + idx;
+ int idxT = idt % 8;
+ int idyT = idt / 8;
+
+ A += gidx*48+ idxT + idyT*lda;
+ B += gidy*48*ldb+ idx + idy*ldb;
+
+
+
+ int block_k = K >> 3;
+ do {
+
+ __local double* plA = lA + idyT*49 + idxT;
+ __local double* plB = lB + idxT*49 + idyT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plA[0] = A[0+0*lda];
+ plA[8] = A[8+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[24] = A[24+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[40] = A[40+0*lda];
+ plB[0] = B[0+0*ldb];
+ plB[8] = B[0+8*ldb];
+ plB[16] = B[0+16*ldb];
+ plB[24] = B[0+24*ldb];
+ plB[32] = B[0+32*ldb];
+ plB[40] = B[0+40*ldb];
+ barrier(CLK_LOCAL_MEM_FENCE);
+ int offA = idx ;
+ int offB = idy ;
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ A += lda << 3;
+ B += 8;
+ } while (--block_k > 0);
+
+
+
+ C+= gidx*48;
+ C+= idx;
+ C+= gidy*48*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[8*ldc] = alpha*rC[0][1] ;
+ C[16*ldc] = alpha*rC[0][2] ;
+ C[24*ldc] = alpha*rC[0][3] ;
+ C[32*ldc] = alpha*rC[0][4] ;
+ C[40*ldc] = alpha*rC[0][5] ;
+ C+=8; ;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[8*ldc] = alpha*rC[1][1] ;
+ C[16*ldc] = alpha*rC[1][2] ;
+ C[24*ldc] = alpha*rC[1][3] ;
+ C[32*ldc] = alpha*rC[1][4] ;
+ C[40*ldc] = alpha*rC[1][5] ;
+ C+=8; ;
+ C[0*ldc] = alpha*rC[2][0] ;
+ C[8*ldc] = alpha*rC[2][1] ;
+ C[16*ldc] = alpha*rC[2][2] ;
+ C[24*ldc] = alpha*rC[2][3] ;
+ C[32*ldc] = alpha*rC[2][4] ;
+ C[40*ldc] = alpha*rC[2][5] ;
+ C+=8; ;
+ C[0*ldc] = alpha*rC[3][0] ;
+ C[8*ldc] = alpha*rC[3][1] ;
+ C[16*ldc] = alpha*rC[3][2] ;
+ C[24*ldc] = alpha*rC[3][3] ;
+ C[32*ldc] = alpha*rC[3][4] ;
+ C[40*ldc] = alpha*rC[3][5] ;
+ C+=8; ;
+ C[0*ldc] = alpha*rC[4][0] ;
+ C[8*ldc] = alpha*rC[4][1] ;
+ C[16*ldc] = alpha*rC[4][2] ;
+ C[24*ldc] = alpha*rC[4][3] ;
+ C[32*ldc] = alpha*rC[4][4] ;
+ C[40*ldc] = alpha*rC[4][5] ;
+ C+=8; ;
+ C[0*ldc] = alpha*rC[5][0] ;
+ C[8*ldc] = alpha*rC[5][1] ;
+ C[16*ldc] = alpha*rC[5][2] ;
+ C[24*ldc] = alpha*rC[5][3] ;
+ C[32*ldc] = alpha*rC[5][4] ;
+ C[40*ldc] = alpha*rC[5][5] ;
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NN_B1_MX048_NX048_KX08_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NN_B1_MX048_NX048_KX08_src.cpp
new file mode 100644
index 0000000..7351ec0
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NN_B1_MX048_NX048_KX08_src.cpp
@@ -0,0 +1,203 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_DGEMM_COL_NN_B1_MX048_NX048_KX08_SRC_H
+#define KERNEL_DGEMM_COL_NN_B1_MX048_NX048_KX08_SRC_H
+#pragma message("AutoGemm's dgemm_Col_NN_B1_MX048_NX048_KX08_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int dgemm_Col_NN_B1_MX048_NX048_KX08_workGroupNumRows = 8;
+const unsigned int dgemm_Col_NN_B1_MX048_NX048_KX08_workGroupNumCols = 8;
+const unsigned int dgemm_Col_NN_B1_MX048_NX048_KX08_microTileNumRows = 6;
+const unsigned int dgemm_Col_NN_B1_MX048_NX048_KX08_microTileNumCols = 6;
+const unsigned int dgemm_Col_NN_B1_MX048_NX048_KX08_unroll = 8;
+
+const char * const dgemm_Col_NN_B1_MX048_NX048_KX08_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0] = lA[offA + 0]; \
+ rA[1] = lA[offA + 8]; \
+ rA[2] = lA[offA + 16]; \
+ rA[3] = lA[offA + 24]; \
+ rA[4] = lA[offA + 32]; \
+ rA[5] = lA[offA + 40]; \
+ rB[0] = lB[offB + 0]; \
+ rB[1] = lB[offB + 8]; \
+ rB[2] = lB[offB + 16]; \
+ rB[3] = lB[offB + 24]; \
+ rB[4] = lB[offB + 32]; \
+ rB[5] = lB[offB + 40]; \
+ offA += 49; \
+ offB += 49; \
+ rC[0][0]=mad(rA[0],rB[0],rC[0][0]); \
+ rC[1][0]=mad(rA[1],rB[0],rC[1][0]); \
+ rC[2][0]=mad(rA[2],rB[0],rC[2][0]); \
+ rC[3][0]=mad(rA[3],rB[0],rC[3][0]); \
+ rC[4][0]=mad(rA[4],rB[0],rC[4][0]); \
+ rC[5][0]=mad(rA[5],rB[0],rC[5][0]); \
+ rC[0][1]=mad(rA[0],rB[1],rC[0][1]); \
+ rC[1][1]=mad(rA[1],rB[1],rC[1][1]); \
+ rC[2][1]=mad(rA[2],rB[1],rC[2][1]); \
+ rC[3][1]=mad(rA[3],rB[1],rC[3][1]); \
+ rC[4][1]=mad(rA[4],rB[1],rC[4][1]); \
+ rC[5][1]=mad(rA[5],rB[1],rC[5][1]); \
+ rC[0][2]=mad(rA[0],rB[2],rC[0][2]); \
+ rC[1][2]=mad(rA[1],rB[2],rC[1][2]); \
+ rC[2][2]=mad(rA[2],rB[2],rC[2][2]); \
+ rC[3][2]=mad(rA[3],rB[2],rC[3][2]); \
+ rC[4][2]=mad(rA[4],rB[2],rC[4][2]); \
+ rC[5][2]=mad(rA[5],rB[2],rC[5][2]); \
+ rC[0][3]=mad(rA[0],rB[3],rC[0][3]); \
+ rC[1][3]=mad(rA[1],rB[3],rC[1][3]); \
+ rC[2][3]=mad(rA[2],rB[3],rC[2][3]); \
+ rC[3][3]=mad(rA[3],rB[3],rC[3][3]); \
+ rC[4][3]=mad(rA[4],rB[3],rC[4][3]); \
+ rC[5][3]=mad(rA[5],rB[3],rC[5][3]); \
+ rC[0][4]=mad(rA[0],rB[4],rC[0][4]); \
+ rC[1][4]=mad(rA[1],rB[4],rC[1][4]); \
+ rC[2][4]=mad(rA[2],rB[4],rC[2][4]); \
+ rC[3][4]=mad(rA[3],rB[4],rC[3][4]); \
+ rC[4][4]=mad(rA[4],rB[4],rC[4][4]); \
+ rC[5][4]=mad(rA[5],rB[4],rC[5][4]); \
+ rC[0][5]=mad(rA[0],rB[5],rC[0][5]); \
+ rC[1][5]=mad(rA[1],rB[5],rC[1][5]); \
+ rC[2][5]=mad(rA[2],rB[5],rC[2][5]); \
+ rC[3][5]=mad(rA[3],rB[5],rC[3][5]); \
+ rC[4][5]=mad(rA[4],rB[5],rC[4][5]); \
+ rC[5][5]=mad(rA[5],rB[5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(8,8,1)))
+__kernel void dgemm_Col_NN_B1_MX048_NX048_KX08 (
+ __global double const * restrict A,
+ __global double const * restrict B,
+ __global double * C,
+ double const alpha,
+ double const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ A += offsetA;
+ B += offsetB;
+ C += offsetC;
+
+
+ double rC[6][6] = {(double)0};
+ double rA[6];
+ double rB[6];
+
+ __local double lA[392];
+ __local double lB[392];
+
+ int gidx = get_group_id(0);
+ int gidy = get_group_id(1);
+ int idx = get_local_id(0);
+ int idy = get_local_id(1);
+
+ int idt = 8*idy + idx;
+ int idxT = idt % 8;
+ int idyT = idt / 8;
+
+ A += gidx*48+ idxT + idyT*lda;
+ B += gidy*48*ldb+ idx + idy*ldb;
+
+
+
+ int block_k = K >> 3;
+ do {
+
+ __local double* plA = lA + idyT*49 + idxT;
+ __local double* plB = lB + idxT*49 + idyT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plA[0] = A[0+0*lda];
+ plA[8] = A[8+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[24] = A[24+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[40] = A[40+0*lda];
+ plB[0] = B[0+0*ldb];
+ plB[8] = B[0+8*ldb];
+ plB[16] = B[0+16*ldb];
+ plB[24] = B[0+24*ldb];
+ plB[32] = B[0+32*ldb];
+ plB[40] = B[0+40*ldb];
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ int offA = idx;
+ int offB = idy;
+
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ A += lda << 3;
+ B += 8;
+ } while (--block_k > 0);
+
+
+
+ C+= gidx*48;
+ C+= idx;
+ C+= gidy*48*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[0][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[0][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[0][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[0][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[0][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[1][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[1][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[1][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[1][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[1][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[2][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[2][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[2][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[2][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[2][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[3][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[3][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[3][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[3][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[3][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[4][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[4][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[4][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[4][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[4][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[5][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[5][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[5][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[5][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[5][5] + beta*C[40*ldc];
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NT_B0_MX048_NX048_KX08_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NT_B0_MX048_NX048_KX08_src.cpp
new file mode 100644
index 0000000..62917b7
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NT_B0_MX048_NX048_KX08_src.cpp
@@ -0,0 +1,196 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_DGEMM_COL_NT_B0_MX048_NX048_KX08_SRC_H
+#define KERNEL_DGEMM_COL_NT_B0_MX048_NX048_KX08_SRC_H
+#pragma message("AutoGemm's dgemm_Col_NT_B0_MX048_NX048_KX08_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int dgemm_Col_NT_B0_MX048_NX048_KX08_workGroupNumRows = 8;
+const unsigned int dgemm_Col_NT_B0_MX048_NX048_KX08_workGroupNumCols = 8;
+const unsigned int dgemm_Col_NT_B0_MX048_NX048_KX08_microTileNumRows = 6;
+const unsigned int dgemm_Col_NT_B0_MX048_NX048_KX08_microTileNumCols = 6;
+const unsigned int dgemm_Col_NT_B0_MX048_NX048_KX08_unroll = 8;
+
+const char * const dgemm_Col_NT_B0_MX048_NX048_KX08_src = STRINGIFY(
+\n
+\ntypedef union _GPtr {
+\n __global float *f;
+\n __global double *d;
+\n __global float2 *f2v;
+\n __global double2 *d2v;
+\n} GPtr;
+\n
+\n#define M6x6 \
+ rA[0] = lA[offA + 0]; \
+ rA[1] = lA[offA + 1]; \
+ rA[2] = lA[offA + 16]; \
+ rA[3] = lA[offA + 17]; \
+ rA[4] = lA[offA + 32]; \
+ rA[5] = lA[offA + 33]; \
+ rB[0] = lB[offB + 0]; \
+ rB[1] = lB[offB + 1]; \
+ rB[2] = lB[offB + 16]; \
+ rB[3] = lB[offB + 17]; \
+ rB[4] = lB[offB + 32]; \
+ rB[5] = lB[offB + 33]; \
+ offA += 48; \
+ offB += 48; \
+ rC[0][0] = mad(rA[0],rB[0],rC[0][0]); \
+ rC[0][1] = mad(rA[1],rB[0],rC[0][1]); \
+ rC[0][2] = mad(rA[2],rB[0],rC[0][2]); \
+ rC[0][3] = mad(rA[3],rB[0],rC[0][3]); \
+ rC[0][4] = mad(rA[4],rB[0],rC[0][4]); \
+ rC[0][5] = mad(rA[5],rB[0],rC[0][5]); \
+ rC[1][0] = mad(rA[0],rB[1],rC[1][0]); \
+ rC[1][1] = mad(rA[1],rB[1],rC[1][1]); \
+ rC[1][2] = mad(rA[2],rB[1],rC[1][2]); \
+ rC[1][3] = mad(rA[3],rB[1],rC[1][3]); \
+ rC[1][4] = mad(rA[4],rB[1],rC[1][4]); \
+ rC[1][5] = mad(rA[5],rB[1],rC[1][5]); \
+ rC[2][0] = mad(rA[0],rB[2],rC[2][0]); \
+ rC[2][1] = mad(rA[1],rB[2],rC[2][1]); \
+ rC[2][2] = mad(rA[2],rB[2],rC[2][2]); \
+ rC[2][3] = mad(rA[3],rB[2],rC[2][3]); \
+ rC[2][4] = mad(rA[4],rB[2],rC[2][4]); \
+ rC[2][5] = mad(rA[5],rB[2],rC[2][5]); \
+ rC[3][0] = mad(rA[0],rB[3],rC[3][0]); \
+ rC[3][1] = mad(rA[1],rB[3],rC[3][1]); \
+ rC[3][2] = mad(rA[2],rB[3],rC[3][2]); \
+ rC[3][3] = mad(rA[3],rB[3],rC[3][3]); \
+ rC[3][4] = mad(rA[4],rB[3],rC[3][4]); \
+ rC[3][5] = mad(rA[5],rB[3],rC[3][5]); \
+ rC[4][0] = mad(rA[0],rB[4],rC[4][0]); \
+ rC[4][1] = mad(rA[1],rB[4],rC[4][1]); \
+ rC[4][2] = mad(rA[2],rB[4],rC[4][2]); \
+ rC[4][3] = mad(rA[3],rB[4],rC[4][3]); \
+ rC[4][4] = mad(rA[4],rB[4],rC[4][4]); \
+ rC[4][5] = mad(rA[5],rB[4],rC[4][5]); \
+ rC[5][0] = mad(rA[0],rB[5],rC[5][0]); \
+ rC[5][1] = mad(rA[1],rB[5],rC[5][1]); \
+ rC[5][2] = mad(rA[2],rB[5],rC[5][2]); \
+ rC[5][3] = mad(rA[3],rB[5],rC[5][3]); \
+ rC[5][4] = mad(rA[4],rB[5],rC[5][4]); \
+ rC[5][5] = mad(rA[5],rB[5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+\n
+\n
+\n__attribute__((reqd_work_group_size(8,8,1)))
+\n__kernel void dgemm_Col_NT_B0_MX048_NX048_KX08 (
+\n __global double2 const * restrict A,
+\n __global double2 const * restrict B,
+\n __global double * C,
+\n double const alpha,
+\n double const beta,
+\n uint const M,
+\n uint const N,
+\n uint const K,
+\n uint lda,
+\n uint ldb,
+\n uint ldc,
+\n uint offsetA,
+\n uint offsetB,
+\n uint offsetC)
+\n{
+\n GPtr uA;
+\n GPtr uB;
+\n uA.d2v = (__global double2 *)A;
+\n uB.d2v = (__global double2 *)B;
+\n
+\n
+\n uA.d += offsetA;
+\n uB.d += offsetB;
+\n C += offsetC;
+\n
+\n
+\n double rC[6][6] = {(double)0};
+\n double rA[6];
+\n double rB[6];
+\n
+\n __local double lA[392];
+\n __local double lB[392];
+\n
+\n int gidx = get_group_id(0);
+\n int gidy = get_group_id(1);
+\n int idx = get_local_id(0);
+\n int idy = get_local_id(1);
+\n
+\n
+\n uA.d += 2*(gidx*24 + idx) + idy*lda;
+\n uB.d += 2*(gidy*24 + idx) + idy*ldb;
+\n
+\n int block_k = K >> 3;
+\n do {
+\n __local double2* plA = (__local double2*)(lA + idy*48 + 2*idx);
+\n __local double2* plB = (__local double2*)(lB + idy*48 + 2*idx);
+\n barrier(CLK_LOCAL_MEM_FENCE);
+\n plB[0 ] = uB.d2v[0 ];
+\n plB[8 ] = uB.d2v[8 ];
+\n plB[16] = uB.d2v[16];
+\n plA[0 ] = uA.d2v[0 ];
+\n plA[8 ] = uA.d2v[8 ];
+\n plA[16] = uA.d2v[16];
+\n
+\n barrier(CLK_LOCAL_MEM_FENCE);
+\n int offA = idx << 1;
+\n int offB = idy << 1;
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n uA.d += lda << 3;
+\n uB.d += ldb << 3;
+\n } while (--block_k > 0);
+\n
+\n
+\n
+\n int offset_x = gidx*48+ idx*2;
+\n int offset_y = gidy*48+ idy*2;
+\n (C[(offset_x + 0) + (offset_y + 0) * ldc] = alpha * rC[0][0]);
+\n (C[(offset_x + 1) + (offset_y + 0) * ldc] = alpha * rC[0][1]);
+\n (C[(offset_x + 0) + (offset_y + 1) * ldc] = alpha * rC[1][0]);
+\n (C[(offset_x + 1) + (offset_y + 1) * ldc] = alpha * rC[1][1]);
+\n (C[(offset_x + 0) + (offset_y + 16) * ldc] = alpha * rC[2][0]);
+\n (C[(offset_x + 1) + (offset_y + 16) * ldc] = alpha * rC[2][1]);
+\n (C[(offset_x + 0) + (offset_y + 17) * ldc] = alpha * rC[3][0]);
+\n (C[(offset_x + 1) + (offset_y + 17) * ldc] = alpha * rC[3][1]);
+\n (C[(offset_x + 0) + (offset_y + 32) * ldc] = alpha * rC[4][0]);
+\n (C[(offset_x + 1) + (offset_y + 32) * ldc] = alpha * rC[4][1]);
+\n (C[(offset_x + 0) + (offset_y + 33) * ldc] = alpha * rC[5][0]);
+\n (C[(offset_x + 1) + (offset_y + 33) * ldc] = alpha * rC[5][1]);
+\n (C[(offset_x + 16) + (offset_y + 0) * ldc] = alpha * rC[0][2]);
+\n (C[(offset_x + 17) + (offset_y + 0) * ldc] = alpha * rC[0][3]);
+\n (C[(offset_x + 16) + (offset_y + 1) * ldc] = alpha * rC[1][2]);
+\n (C[(offset_x + 17) + (offset_y + 1) * ldc] = alpha * rC[1][3]);
+\n (C[(offset_x + 16) + (offset_y + 16) * ldc] = alpha * rC[2][2]);
+\n (C[(offset_x + 17) + (offset_y + 16) * ldc] = alpha * rC[2][3]);
+\n (C[(offset_x + 16) + (offset_y + 17) * ldc] = alpha * rC[3][2]);
+\n (C[(offset_x + 17) + (offset_y + 17) * ldc] = alpha * rC[3][3]);
+\n (C[(offset_x + 16) + (offset_y + 32) * ldc] = alpha * rC[4][2]);
+\n (C[(offset_x + 17) + (offset_y + 32) * ldc] = alpha * rC[4][3]);
+\n (C[(offset_x + 16) + (offset_y + 33) * ldc] = alpha * rC[5][2]);
+\n (C[(offset_x + 17) + (offset_y + 33) * ldc] = alpha * rC[5][3]);
+\n (C[(offset_x + 32) + (offset_y + 0) * ldc] = alpha * rC[0][4]);
+\n (C[(offset_x + 33) + (offset_y + 0) * ldc] = alpha * rC[0][5]);
+\n (C[(offset_x + 32) + (offset_y + 1) * ldc] = alpha * rC[1][4]);
+\n (C[(offset_x + 33) + (offset_y + 1) * ldc] = alpha * rC[1][5]);
+\n (C[(offset_x + 32) + (offset_y + 16) * ldc] = alpha * rC[2][4]);
+\n (C[(offset_x + 33) + (offset_y + 16) * ldc] = alpha * rC[2][5]);
+\n (C[(offset_x + 32) + (offset_y + 17) * ldc] = alpha * rC[3][4]);
+\n (C[(offset_x + 33) + (offset_y + 17) * ldc] = alpha * rC[3][5]);
+\n (C[(offset_x + 32) + (offset_y + 32) * ldc] = alpha * rC[4][4]);
+\n (C[(offset_x + 33) + (offset_y + 32) * ldc] = alpha * rC[4][5]);
+\n (C[(offset_x + 32) + (offset_y + 33) * ldc] = alpha * rC[5][4]);
+\n (C[(offset_x + 33) + (offset_y + 33) * ldc] = alpha * rC[5][5]);
+\n}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NT_B1_MX048_NX048_KX08_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NT_B1_MX048_NX048_KX08_src.cpp
new file mode 100644
index 0000000..3026a0e
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_NT_B1_MX048_NX048_KX08_src.cpp
@@ -0,0 +1,193 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_DGEMM_COL_NT_B1_MX048_NX048_KX08_SRC_H
+#define KERNEL_DGEMM_COL_NT_B1_MX048_NX048_KX08_SRC_H
+#pragma message("AutoGemm's dgemm_Col_NT_B1_MX048_NX048_KX08_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int dgemm_Col_NT_B1_MX048_NX048_KX08_workGroupNumRows = 8;
+const unsigned int dgemm_Col_NT_B1_MX048_NX048_KX08_workGroupNumCols = 8;
+const unsigned int dgemm_Col_NT_B1_MX048_NX048_KX08_microTileNumRows = 6;
+const unsigned int dgemm_Col_NT_B1_MX048_NX048_KX08_microTileNumCols = 6;
+const unsigned int dgemm_Col_NT_B1_MX048_NX048_KX08_unroll = 8;
+
+const char * const dgemm_Col_NT_B1_MX048_NX048_KX08_src = STRINGIFY(
+\n
+\ntypedef union _GPtr {
+\n __global float *f;
+\n __global double *d;
+\n __global float2 *f2v;
+\n __global double2 *d2v;
+\n} GPtr;
+\n
+\n#define M6x6 \
+ rA[0] = lA[offA + 0]; \
+ rA[1] = lA[offA + 1]; \
+ rA[2] = lA[offA + 16]; \
+ rA[3] = lA[offA + 17]; \
+ rA[4] = lA[offA + 32]; \
+ rA[5] = lA[offA + 33]; \
+ rB[0] = lB[offB + 0]; \
+ rB[1] = lB[offB + 1]; \
+ rB[2] = lB[offB + 16]; \
+ rB[3] = lB[offB + 17]; \
+ rB[4] = lB[offB + 32]; \
+ rB[5] = lB[offB + 33]; \
+ offA += 48; \
+ offB += 48; \
+ rC[0][0] = mad(rA[0],rB[0],rC[0][0]); \
+ rC[0][1] = mad(rA[1],rB[0],rC[0][1]); \
+ rC[0][2] = mad(rA[2],rB[0],rC[0][2]); \
+ rC[0][3] = mad(rA[3],rB[0],rC[0][3]); \
+ rC[0][4] = mad(rA[4],rB[0],rC[0][4]); \
+ rC[0][5] = mad(rA[5],rB[0],rC[0][5]); \
+ rC[1][0] = mad(rA[0],rB[1],rC[1][0]); \
+ rC[1][1] = mad(rA[1],rB[1],rC[1][1]); \
+ rC[1][2] = mad(rA[2],rB[1],rC[1][2]); \
+ rC[1][3] = mad(rA[3],rB[1],rC[1][3]); \
+ rC[1][4] = mad(rA[4],rB[1],rC[1][4]); \
+ rC[1][5] = mad(rA[5],rB[1],rC[1][5]); \
+ rC[2][0] = mad(rA[0],rB[2],rC[2][0]); \
+ rC[2][1] = mad(rA[1],rB[2],rC[2][1]); \
+ rC[2][2] = mad(rA[2],rB[2],rC[2][2]); \
+ rC[2][3] = mad(rA[3],rB[2],rC[2][3]); \
+ rC[2][4] = mad(rA[4],rB[2],rC[2][4]); \
+ rC[2][5] = mad(rA[5],rB[2],rC[2][5]); \
+ rC[3][0] = mad(rA[0],rB[3],rC[3][0]); \
+ rC[3][1] = mad(rA[1],rB[3],rC[3][1]); \
+ rC[3][2] = mad(rA[2],rB[3],rC[3][2]); \
+ rC[3][3] = mad(rA[3],rB[3],rC[3][3]); \
+ rC[3][4] = mad(rA[4],rB[3],rC[3][4]); \
+ rC[3][5] = mad(rA[5],rB[3],rC[3][5]); \
+ rC[4][0] = mad(rA[0],rB[4],rC[4][0]); \
+ rC[4][1] = mad(rA[1],rB[4],rC[4][1]); \
+ rC[4][2] = mad(rA[2],rB[4],rC[4][2]); \
+ rC[4][3] = mad(rA[3],rB[4],rC[4][3]); \
+ rC[4][4] = mad(rA[4],rB[4],rC[4][4]); \
+ rC[4][5] = mad(rA[5],rB[4],rC[4][5]); \
+ rC[5][0] = mad(rA[0],rB[5],rC[5][0]); \
+ rC[5][1] = mad(rA[1],rB[5],rC[5][1]); \
+ rC[5][2] = mad(rA[2],rB[5],rC[5][2]); \
+ rC[5][3] = mad(rA[3],rB[5],rC[5][3]); \
+ rC[5][4] = mad(rA[4],rB[5],rC[5][4]); \
+ rC[5][5] = mad(rA[5],rB[5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+\n
+\n
+\n__attribute__((reqd_work_group_size(8,8,1)))
+\n__kernel void dgemm_Col_NT_B1_MX048_NX048_KX08_src (
+\n __global double2 const * restrict A,
+\n __global double2 const * restrict B,
+\n __global double * C,
+\n double const alpha,
+\n double const beta,
+\n uint const M,
+\n uint const N,
+\n uint const K,
+\n uint lda,
+\n uint ldb,
+\n uint ldc,
+\n uint offsetA,
+\n uint offsetB,
+\n uint offsetC)
+\n{
+\n GPtr uA;
+\n GPtr uB;
+\n uA.d2v = (__global double2 *)A;
+\n uB.d2v = (__global double2 *)B;
+\n
+\n uA.d += offsetA;
+\n uB.d += offsetB;
+\n C += offsetC;
+\n
+\n
+\n double rC[6][6] = {(double)0};
+\n double rA[6];
+\n double rB[6];
+\n
+\n __local double lA[392];
+\n __local double lB[392];
+\n
+\n int gidx = get_group_id(0);
+\n int gidy = get_group_id(1);
+\n int idx = get_local_id(0);
+\n int idy = get_local_id(1);
+\n
+\n
+\n uA.d += 2*(gidx*24 + idx) + idy*lda;
+\n uB.d += 2*(gidy*24 + idx) + idy*ldb;
+\n
+\n int block_k = K >> 3;
+\n do {
+\n __local double2* plA = (__local double2*)(lA + idy*48 + 2*idx);
+\n __local double2* plB = (__local double2*)(lB + idy*48 + 2*idx);
+\n barrier(CLK_LOCAL_MEM_FENCE);
+\n plB[0 ] = uB.d2v[0 ];
+\n plB[8 ] = uB.d2v[8 ];
+\n plB[16] = uB.d2v[16];
+\n plA[0 ] = uA.d2v[0 ];
+\n plA[8 ] = uA.d2v[8 ];
+\n plA[16] = uA.d2v[16];
+\n
+\n barrier(CLK_LOCAL_MEM_FENCE);
+\n int offA = idx << 1;
+\n int offB = idy << 1;
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n M6x6
+\n uA.d += lda << 3;
+\n uB.d += ldb << 3;
+\n } while (--block_k > 0);
+\n
+\n int offset_x = gidx*48+ idx*2;
+\n int offset_y = gidy*48+ idy*2;
+\n (C[(offset_x + 0) + (offset_y + 0) * ldc] = mad(beta, C[(offset_x + 0) + (offset_y + 0) * ldc], alpha * rC[0][0]));
+\n (C[(offset_x + 1) + (offset_y + 0) * ldc] = mad(beta, C[(offset_x + 1) + (offset_y + 0) * ldc], alpha * rC[0][1]));
+\n (C[(offset_x + 0) + (offset_y + 1) * ldc] = mad(beta, C[(offset_x + 0) + (offset_y + 1) * ldc], alpha * rC[1][0]));
+\n (C[(offset_x + 1) + (offset_y + 1) * ldc] = mad(beta, C[(offset_x + 1) + (offset_y + 1) * ldc], alpha * rC[1][1]));
+\n (C[(offset_x + 0) + (offset_y + 16) * ldc] = mad(beta, C[(offset_x + 0) + (offset_y + 16) * ldc], alpha * rC[2][0]));
+\n (C[(offset_x + 1) + (offset_y + 16) * ldc] = mad(beta, C[(offset_x + 1) + (offset_y + 16) * ldc], alpha * rC[2][1]));
+\n (C[(offset_x + 0) + (offset_y + 17) * ldc] = mad(beta, C[(offset_x + 0) + (offset_y + 17) * ldc], alpha * rC[3][0]));
+\n (C[(offset_x + 1) + (offset_y + 17) * ldc] = mad(beta, C[(offset_x + 1) + (offset_y + 17) * ldc], alpha * rC[3][1]));
+\n (C[(offset_x + 0) + (offset_y + 32) * ldc] = mad(beta, C[(offset_x + 0) + (offset_y + 32) * ldc], alpha * rC[4][0]));
+\n (C[(offset_x + 1) + (offset_y + 32) * ldc] = mad(beta, C[(offset_x + 1) + (offset_y + 32) * ldc], alpha * rC[4][1]));
+\n (C[(offset_x + 0) + (offset_y + 33) * ldc] = mad(beta, C[(offset_x + 0) + (offset_y + 33) * ldc], alpha * rC[5][0]));
+\n (C[(offset_x + 1) + (offset_y + 33) * ldc] = mad(beta, C[(offset_x + 1) + (offset_y + 33) * ldc], alpha * rC[5][1]));
+\n (C[(offset_x + 16) + (offset_y + 0) * ldc] = mad(beta, C[(offset_x + 16) + (offset_y + 0) * ldc], alpha * rC[0][2]));
+\n (C[(offset_x + 17) + (offset_y + 0) * ldc] = mad(beta, C[(offset_x + 17) + (offset_y + 0) * ldc], alpha * rC[0][3]));
+\n (C[(offset_x + 16) + (offset_y + 1) * ldc] = mad(beta, C[(offset_x + 16) + (offset_y + 1) * ldc], alpha * rC[1][2]));
+\n (C[(offset_x + 17) + (offset_y + 1) * ldc] = mad(beta, C[(offset_x + 17) + (offset_y + 1) * ldc], alpha * rC[1][3]));
+\n (C[(offset_x + 16) + (offset_y + 16) * ldc] = mad(beta, C[(offset_x + 16) + (offset_y + 16) * ldc], alpha * rC[2][2]));
+\n (C[(offset_x + 17) + (offset_y + 16) * ldc] = mad(beta, C[(offset_x + 17) + (offset_y + 16) * ldc], alpha * rC[2][3]));
+\n (C[(offset_x + 16) + (offset_y + 17) * ldc] = mad(beta, C[(offset_x + 16) + (offset_y + 17) * ldc], alpha * rC[3][2]));
+\n (C[(offset_x + 17) + (offset_y + 17) * ldc] = mad(beta, C[(offset_x + 17) + (offset_y + 17) * ldc], alpha * rC[3][3]));
+\n (C[(offset_x + 16) + (offset_y + 32) * ldc] = mad(beta, C[(offset_x + 16) + (offset_y + 32) * ldc], alpha * rC[4][2]));
+\n (C[(offset_x + 17) + (offset_y + 32) * ldc] = mad(beta, C[(offset_x + 17) + (offset_y + 32) * ldc], alpha * rC[4][3]));
+\n (C[(offset_x + 16) + (offset_y + 33) * ldc] = mad(beta, C[(offset_x + 16) + (offset_y + 33) * ldc], alpha * rC[5][2]));
+\n (C[(offset_x + 17) + (offset_y + 33) * ldc] = mad(beta, C[(offset_x + 17) + (offset_y + 33) * ldc], alpha * rC[5][3]));
+\n (C[(offset_x + 32) + (offset_y + 0) * ldc] = mad(beta, C[(offset_x + 32) + (offset_y + 0) * ldc], alpha * rC[0][4]));
+\n (C[(offset_x + 33) + (offset_y + 0) * ldc] = mad(beta, C[(offset_x + 33) + (offset_y + 0) * ldc], alpha * rC[0][5]));
+\n (C[(offset_x + 32) + (offset_y + 1) * ldc] = mad(beta, C[(offset_x + 32) + (offset_y + 1) * ldc], alpha * rC[1][4]));
+\n (C[(offset_x + 33) + (offset_y + 1) * ldc] = mad(beta, C[(offset_x + 33) + (offset_y + 1) * ldc], alpha * rC[1][5]));
+\n (C[(offset_x + 32) + (offset_y + 16) * ldc] = mad(beta, C[(offset_x + 32) + (offset_y + 16) * ldc], alpha * rC[2][4]));
+\n (C[(offset_x + 33) + (offset_y + 16) * ldc] = mad(beta, C[(offset_x + 33) + (offset_y + 16) * ldc], alpha * rC[2][5]));
+\n (C[(offset_x + 32) + (offset_y + 17) * ldc] = mad(beta, C[(offset_x + 32) + (offset_y + 17) * ldc], alpha * rC[3][4]));
+\n (C[(offset_x + 33) + (offset_y + 17) * ldc] = mad(beta, C[(offset_x + 33) + (offset_y + 17) * ldc], alpha * rC[3][5]));
+\n (C[(offset_x + 32) + (offset_y + 32) * ldc] = mad(beta, C[(offset_x + 32) + (offset_y + 32) * ldc], alpha * rC[4][4]));
+\n (C[(offset_x + 33) + (offset_y + 32) * ldc] = mad(beta, C[(offset_x + 33) + (offset_y + 32) * ldc], alpha * rC[4][5]));
+\n (C[(offset_x + 32) + (offset_y + 33) * ldc] = mad(beta, C[(offset_x + 32) + (offset_y + 33) * ldc], alpha * rC[5][4]));
+\n (C[(offset_x + 33) + (offset_y + 33) * ldc] = mad(beta, C[(offset_x + 33) + (offset_y + 33) * ldc], alpha * rC[5][5]));
+\n}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_TN_B0_MX048_NX048_KX08_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_TN_B0_MX048_NX048_KX08_src.cpp
new file mode 100644
index 0000000..1c0684b
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_TN_B0_MX048_NX048_KX08_src.cpp
@@ -0,0 +1,195 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_DGEMM_COL_TN_B0_MX048_NX048_KX08_SRC_H
+#define KERNEL_DGEMM_COL_TN_B0_MX048_NX048_KX08_SRC_H
+#pragma message("AutoGemm's dgemm_Col_TN_B0_MX048_NX048_KX08_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int dgemm_Col_TN_B0_MX048_NX048_KX08_workGroupNumRows = 8;
+const unsigned int dgemm_Col_TN_B0_MX048_NX048_KX08_workGroupNumCols = 8;
+const unsigned int dgemm_Col_TN_B0_MX048_NX048_KX08_microTileNumRows = 6;
+const unsigned int dgemm_Col_TN_B0_MX048_NX048_KX08_microTileNumCols = 6;
+const unsigned int dgemm_Col_TN_B0_MX048_NX048_KX08_unroll = 8;
+
+const char * const dgemm_Col_TN_B0_MX048_NX048_KX08_src = STRINGIFY(
+
+__attribute__( (reqd_work_group_size(8, 8, 1)) )
+__kernel void dgemm_Col_TN_B0_MX048_NX048_KX08_src (
+ __global double const * restrict A,
+ __global double const * restrict B,
+ __global double * C,
+ double const alpha,
+ double const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint const offsetA,
+ uint const offsetB,
+ uint const offsetC )
+{
+ double rC[6][6] = {(double)0};
+ double rA[1][6];
+ double rB[1][6];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local double lA[392];
+ __local double lB[392];
+
+ int gidx = get_group_id(0);
+ int gidy = get_group_id(1);
+ int idx = get_local_id(0);
+ int idy = get_local_id(1);
+
+ int idt = 8*idy + idx;
+ int idxT = idt % 4;
+ int idyT = idt / 4;
+
+ A += gidx*48*lda + idxT + idyT*lda;
+ B += gidy*48*ldb+ idxT + idyT*ldb;
+
+ //for( int block_k=0 ; block_k< K ; block_k+=8)
+ uint block_k = K >> 3;
+ do
+ {
+ __local double* plA = lA + idxT*49+ idyT;
+ __local double* plB = lB + idxT*49+ idyT;
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plA[0] = A[0];
+ plA[196] = A[4];
+ plA[16] = A[16*lda];
+ plA[212] = A[4+16*lda];
+ plA[32] = A[32*lda];
+ plA[228] = A[4+32*lda];
+ plB[0] = B[0];
+ plB[196] = B[4+0*ldb];
+ plB[16] = B[0+16*ldb];
+ plB[212] = B[4+16*ldb];
+ plB[32] = B[0+32*ldb];
+ plB[228] = B[4+32*ldb];
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ int offA = 1*idx;
+ int offB = 1*idy;
+
+ for( int k = 0 ; k < 8; k+=1)
+ {
+ rA[0][0] = lA[offA + 0];
+ rA[0][1] = lA[offA + 8];
+ rA[0][2] = lA[offA + 16];
+ rA[0][3] = lA[offA + 24];
+ rA[0][4] = lA[offA + 32];
+ rA[0][5] = lA[offA + 40];
+ rB[0][0] = lB[offB + 0];
+ rB[0][1] = lB[offB + 8];
+ rB[0][2] = lB[offB + 16];
+ rB[0][3] = lB[offB + 24];
+ rB[0][4] = lB[offB + 32];
+ rB[0][5] = lB[offB + 40];
+ offA += 49;
+ offB += 49;
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]);
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]);
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]);
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]);
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]);
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]);
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]);
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]);
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]);
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]);
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]);
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]);
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]);
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]);
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]);
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]);
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]);
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]);
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]);
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]);
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]);
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]);
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]);
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]);
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]);
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]);
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]);
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]);
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]);
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]);
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]);
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]);
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]);
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]);
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]);
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]);
+ }
+ A += 8;
+ B += 8;
+ }
+ while (--block_k > 0);
+
+ C+= gidx*48;
+ C+= idx;
+ C+= gidy*48*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[8*ldc] = alpha*rC[0][1] ;
+ C[16*ldc] = alpha*rC[0][2];
+ C[24*ldc] = alpha*rC[0][3];
+ C[32*ldc] = alpha*rC[0][4];
+ C[40*ldc] = alpha*rC[0][5];
+ C+=8;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[8*ldc] = alpha*rC[1][1] ;
+ C[16*ldc] = alpha*rC[1][2];
+ C[24*ldc] = alpha*rC[1][3];
+ C[32*ldc] = alpha*rC[1][4];
+ C[40*ldc] = alpha*rC[1][5];
+ C+=8;
+ C[0*ldc] = alpha*rC[2][0] ;
+ C[8*ldc] = alpha*rC[2][1] ;
+ C[16*ldc] = alpha*rC[2][2];
+ C[24*ldc] = alpha*rC[2][3];
+ C[32*ldc] = alpha*rC[2][4];
+ C[40*ldc] = alpha*rC[2][5];
+ C+=8;
+ C[0*ldc] = alpha*rC[3][0] ;
+ C[8*ldc] = alpha*rC[3][1] ;
+ C[16*ldc] = alpha*rC[3][2];
+ C[24*ldc] = alpha*rC[3][3];
+ C[32*ldc] = alpha*rC[3][4];
+ C[40*ldc] = alpha*rC[3][5];
+ C+=8;
+ C[0*ldc] = alpha*rC[4][0] ;
+ C[8*ldc] = alpha*rC[4][1] ;
+ C[16*ldc] = alpha*rC[4][2];
+ C[24*ldc] = alpha*rC[4][3];
+ C[32*ldc] = alpha*rC[4][4];
+ C[40*ldc] = alpha*rC[4][5];
+ C+=8;
+ C[0*ldc] = alpha*rC[5][0] ;
+ C[8*ldc] = alpha*rC[5][1] ;
+ C[16*ldc] = alpha*rC[5][2];
+ C[24*ldc] = alpha*rC[5][3];
+ C[32*ldc] = alpha*rC[5][4];
+ C[40*ldc] = alpha*rC[5][5];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_TN_B1_MX048_NX048_KX08_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_TN_B1_MX048_NX048_KX08_src.cpp
new file mode 100644
index 0000000..5902469
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/dgemm_Col_TN_B1_MX048_NX048_KX08_src.cpp
@@ -0,0 +1,195 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_DGEMM_COL_TN_B1_MX048_NX048_KX08_SRC_H
+#define KERNEL_DGEMM_COL_TN_B1_MX048_NX048_KX08_SRC_H
+#pragma message("AutoGemm's dgemm_Col_TN_B1_MX048_NX048_KX08_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int dgemm_Col_TN_B1_MX048_NX048_KX08_workGroupNumRows = 8;
+const unsigned int dgemm_Col_TN_B1_MX048_NX048_KX08_workGroupNumCols = 8;
+const unsigned int dgemm_Col_TN_B1_MX048_NX048_KX08_microTileNumRows = 6;
+const unsigned int dgemm_Col_TN_B1_MX048_NX048_KX08_microTileNumCols = 6;
+const unsigned int dgemm_Col_TN_B1_MX048_NX048_KX08_unroll = 8;
+
+const char * const dgemm_Col_TN_B1_MX048_NX048_KX08_src = STRINGIFY(
+
+__attribute__( (reqd_work_group_size(8, 8, 1)) )
+__kernel void dgemm_Col_TN_B1_MX048_NX048_KX08_src (
+ __global double const * restrict A,
+ __global double const * restrict B,
+ __global double * C,
+ double const alpha,
+ double const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint const offsetA,
+ uint const offsetB,
+ uint const offsetC )
+{
+ double rC[6][6] = {(double)0};
+ double rA[1][6];
+ double rB[1][6];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local double lA[392];
+ __local double lB[392];
+
+ int gidx = get_group_id(0);
+ int gidy = get_group_id(1);
+ int idx = get_local_id(0);
+ int idy = get_local_id(1);
+
+ int idt = 8*idy + idx;
+ int idxT = idt % 4;
+ int idyT = idt / 4;
+
+ A += gidx*48*lda + idxT + idyT*lda;
+ B += gidy*48*ldb+ idxT + idyT*ldb;
+
+ //for( int block_k=0 ; block_k< K ; block_k+=8)
+ uint block_k = K >> 3;
+ do
+ {
+ __local double* plA = lA + idxT*49+ idyT;
+ __local double* plB = lB + idxT*49+ idyT;
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plA[0] = A[0];
+ plA[196] = A[4];
+ plA[16] = A[16*lda];
+ plA[212] = A[4+16*lda];
+ plA[32] = A[32*lda];
+ plA[228] = A[4+32*lda];
+ plB[0] = B[0];
+ plB[196] = B[4+0*ldb];
+ plB[16] = B[0+16*ldb];
+ plB[212] = B[4+16*ldb];
+ plB[32] = B[0+32*ldb];
+ plB[228] = B[4+32*ldb];
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ int offA = 1*idx;
+ int offB = 1*idy;
+
+ for( int k = 0 ; k < 8; k+=1)
+ {
+ rA[0][0] = lA[offA + 0];
+ rA[0][1] = lA[offA + 8];
+ rA[0][2] = lA[offA + 16];
+ rA[0][3] = lA[offA + 24];
+ rA[0][4] = lA[offA + 32];
+ rA[0][5] = lA[offA + 40];
+ rB[0][0] = lB[offB + 0];
+ rB[0][1] = lB[offB + 8];
+ rB[0][2] = lB[offB + 16];
+ rB[0][3] = lB[offB + 24];
+ rB[0][4] = lB[offB + 32];
+ rB[0][5] = lB[offB + 40];
+ offA += 49;
+ offB += 49;
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]);
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]);
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]);
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]);
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]);
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]);
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]);
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]);
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]);
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]);
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]);
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]);
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]);
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]);
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]);
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]);
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]);
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]);
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]);
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]);
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]);
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]);
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]);
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]);
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]);
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]);
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]);
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]);
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]);
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]);
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]);
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]);
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]);
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]);
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]);
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]);
+ }
+ A += 8;
+ B += 8;
+ }
+ while (--block_k > 0);
+
+ C+= gidx*48;
+ C+= idx;
+ C+= gidy*48*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[0][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[0][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[0][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[0][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[0][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[1][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[1][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[1][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[1][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[1][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[2][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[2][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[2][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[2][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[2][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[3][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[3][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[3][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[3][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[3][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[4][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[4][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[4][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[4][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[4][5] + beta*C[40*ldc];
+ C+=8;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[8*ldc] = alpha*rC[5][1] + beta*C[8*ldc];
+ C[16*ldc] = alpha*rC[5][2] + beta*C[16*ldc];
+ C[24*ldc] = alpha*rC[5][3] + beta*C[24*ldc];
+ C[32*ldc] = alpha*rC[5][4] + beta*C[32*ldc];
+ C[40*ldc] = alpha*rC[5][5] + beta*C[40*ldc];
+ C+=8;
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX032_NX032_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX032_NX032_KX16_src.cpp
new file mode 100644
index 0000000..9af1029
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX032_NX032_KX16_src.cpp
@@ -0,0 +1,129 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NN_B0_MX032_NX032_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NN_B0_MX032_NX032_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NN_B0_MX032_NX032_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NN_B0_MX032_NX032_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NN_B0_MX032_NX032_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NN_B0_MX032_NX032_KX16_microTileNumRows = 2;
+const unsigned int sgemm_Col_NN_B0_MX032_NX032_KX16_microTileNumCols = 2;
+const unsigned int sgemm_Col_NN_B0_MX032_NX032_KX16_unroll = 16;
+
+const char * const sgemm_Col_NN_B0_MX032_NX032_KX16_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NN_B0_MX032_NX032_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[544];
+ __local float lB[544];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*32+ idxT + idyT*lda;
+ B += (gidy*32+idyT)*ldb + idxT;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idyT*33+idxT;
+ __local float* plB = lB + idxT*33+idyT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += lda<<4;
+ B += 16;
+
+ } while (--block_k > 0);
+
+ C+= gidx*32+idx;
+ C+= gidy*32*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[16*ldc] = alpha*rC[0][1];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[16*ldc] = alpha*rC[1][1];
+}
+
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX064_NX064_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX064_NX064_KX16_src.cpp
new file mode 100644
index 0000000..8e60127
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX064_NX064_KX16_src.cpp
@@ -0,0 +1,160 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NN_B0_MX064_NX064_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NN_B0_MX064_NX064_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NN_B0_MX064_NX064_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NN_B0_MX064_NX064_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NN_B0_MX064_NX064_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NN_B0_MX064_NX064_KX16_microTileNumRows = 4;
+const unsigned int sgemm_Col_NN_B0_MX064_NX064_KX16_microTileNumCols = 4;
+const unsigned int sgemm_Col_NN_B0_MX064_NX064_KX16_unroll = 16;
+
+const char * const sgemm_Col_NN_B0_MX064_NX064_KX16_src = STRINGIFY(
+
+#define M4x4 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ offA += 65; \
+ offB += 65; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NN_B0_MX064_NX064_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[4][4] = {(float)0};
+ float rA[1][4];
+ float rB[1][4];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1056];
+ __local float lB[1056];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += gidx*64+ idx + idy*lda;
+ B += (gidy*64+idy)*ldb+ idx;
+
+
+ uint block_k = K >> 4;
+ do {
+ __local float* plA = lA + idy*65+idx;
+ __local float* plB = lB + idx*65+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+
+ A += lda<<4;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*64+idx;
+ C+= gidy*64*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[16*ldc] = alpha*rC[0][1];
+ C[32*ldc] = alpha*rC[0][2];
+ C[48*ldc] = alpha*rC[0][3];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[16*ldc] = alpha*rC[1][1];
+ C[32*ldc] = alpha*rC[1][2];
+ C[48*ldc] = alpha*rC[1][3];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] ;
+ C[16*ldc] = alpha*rC[2][1];
+ C[32*ldc] = alpha*rC[2][2];
+ C[48*ldc] = alpha*rC[2][3];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] ;
+ C[16*ldc] = alpha*rC[3][1];
+ C[32*ldc] = alpha*rC[3][2];
+ C[48*ldc] = alpha*rC[3][3];
+
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX096_NX096_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX096_NX096_KX16_src.cpp
new file mode 100644
index 0000000..8bb193e
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B0_MX096_NX096_KX16_src.cpp
@@ -0,0 +1,208 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NN_B0_MX096_NX096_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NN_B0_MX096_NX096_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NN_B0_MX096_NX096_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NN_B0_MX096_NX096_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NN_B0_MX096_NX096_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NN_B0_MX096_NX096_KX16_microTileNumRows = 6;
+const unsigned int sgemm_Col_NN_B0_MX096_NX096_KX16_microTileNumCols = 6;
+const unsigned int sgemm_Col_NN_B0_MX096_NX096_KX16_unroll = 16;
+
+const char * const sgemm_Col_NN_B0_MX096_NX096_KX16_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rA[0][4] = lA[offA + 64]; \
+ rA[0][5] = lA[offA + 80]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ rB[0][4] = lB[offB + 64]; \
+ rB[0][5] = lB[offB + 80]; \
+ offA += 97; \
+ offB += 97; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]); \
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]); \
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]); \
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]); \
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]); \
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]); \
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]); \
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]); \
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]); \
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]); \
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]); \
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]); \
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]); \
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]); \
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]); \
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]); \
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NN_B0_MX096_NX096_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[6][6] = {(float)0};
+ float rA[1][6];
+ float rB[1][6];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1552];
+ __local float lB[1552];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += gidx*96+ idx + idy*lda;
+ B += gidy*96*ldb+ idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy*97+idx;
+ __local float* plB = lB + idx*97+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+ plB[64] = B[64*ldb];
+ plB[80] = B[80*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+ plA[64] = A[64+0*lda];
+ plA[80] = A[80+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+
+ A += lda<<4;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*96+idx;
+ C+= gidy*96*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[0][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[0][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[1][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[1][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[2][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[2][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[3][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[3][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[4][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[4][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[4][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[4][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[4][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[5][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[5][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[5][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[5][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[5][5] + beta*C[80*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src.cpp
new file mode 100644
index 0000000..d45b41d
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src.cpp
@@ -0,0 +1,149 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NN_B1_MX032_NX032_KX16_BRANCH_SRC_H
+#define KERNEL_SGEMM_COL_NN_B1_MX032_NX032_KX16_BRANCH_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src (if exists) overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_microTileNumRows = 2;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_microTileNumCols = 2;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_unroll = 16;
+
+#ifndef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+unsigned char *sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_bin = 0;
+size_t sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_binSize = 0;
+#endif
+
+const char * const sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[528];//16*32+16
+ __local float lB[528];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ int CurrentOffSetA = gidx*32+ idx;
+ int CurrentOffSetB = gidy*32+ idy;
+
+ A += gidx*32+ idx + idy*lda;
+ B += gidy*32*ldb+ idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy*33+idx;
+ __local float* plB = lB + idx*33+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ plB[0] = CurrentOffSetB>=N?0.0:B[0];
+ plB[16] = CurrentOffSetB+16>=N?0.0:B[16*ldb];
+
+ plA[0] = CurrentOffSetA>=M?0.0:A[0];
+ plA[16] = CurrentOffSetA+16>=M?0.0:A[16];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += lda<<4;
+ B += 16;
+ //}
+ } while (--block_k > 0);
+
+ int offset_x = gidx*32+idx;
+ int offset_y = gidy*32+ idy;
+ if(offset_x>=M || offset_y>=N )
+ return;
+
+ C+=offset_x+offset_y*ldc;
+
+
+ int i = 0;
+ do
+ {
+ C[0 ] = mad(alpha, rC[i][0], beta*C[0]);
+ if(offset_y+16<N)
+ C[16*ldc] = mad(alpha, rC[i][1], beta*C[16*ldc]);
+
+ C+=16;
+ offset_x+=16;
+ if(offset_x>=M )
+ return;
+
+
+ }
+ while (++i < 2);
+
+}
+
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_src.cpp
new file mode 100644
index 0000000..573827c
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX032_NX032_KX16_src.cpp
@@ -0,0 +1,129 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NN_B1_MX032_NX032_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NN_B1_MX032_NX032_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NN_B1_MX032_NX032_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_microTileNumRows = 2;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_microTileNumCols = 2;
+const unsigned int sgemm_Col_NN_B1_MX032_NX032_KX16_unroll = 16;
+
+const char * const sgemm_Col_NN_B1_MX032_NX032_KX16_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NN_B1_MX032_NX032_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[544];
+ __local float lB[544];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*32+ idxT + idyT*lda;
+ B += (gidy*32+idyT)*ldb + idxT;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+
+ __local float* plA = lA + idyT*33+idxT;
+ __local float* plB = lB + idxT*33+idyT;
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += lda<<4;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*32+idx;
+ C+= gidy*32*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+}
+
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX064_NX064_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX064_NX064_KX16_src.cpp
new file mode 100644
index 0000000..602d28f
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX064_NX064_KX16_src.cpp
@@ -0,0 +1,161 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NN_B1_MX064_NX064_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NN_B1_MX064_NX064_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NN_B1_MX064_NX064_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NN_B1_MX064_NX064_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NN_B1_MX064_NX064_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NN_B1_MX064_NX064_KX16_microTileNumRows = 4;
+const unsigned int sgemm_Col_NN_B1_MX064_NX064_KX16_microTileNumCols = 4;
+const unsigned int sgemm_Col_NN_B1_MX064_NX064_KX16_unroll = 16;
+
+const char * const sgemm_Col_NN_B1_MX064_NX064_KX16_src = STRINGIFY(
+
+#define M4x4 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ offA += 65; \
+ offB += 65; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NN_B1_MX064_NX064_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[4][4] = {(float)0};
+ float rA[1][4];
+ float rB[1][4];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1056];
+ __local float lB[1056];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += gidx*64+ idx + idy*lda;
+ B += (gidy*64+idy)*ldb+ idx;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy*65+idx;
+ __local float* plB = lB + idx*65+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+
+ A += lda<<4;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*64+idx;
+ C+= gidy*64*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX096_NX096_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX096_NX096_KX16_src.cpp
new file mode 100644
index 0000000..e5795c9
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NN_B1_MX096_NX096_KX16_src.cpp
@@ -0,0 +1,207 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NN_B1_MX096_NX096_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NN_B1_MX096_NX096_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NN_B1_MX096_NX096_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NN_B1_MX096_NX096_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NN_B1_MX096_NX096_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NN_B1_MX096_NX096_KX16_microTileNumRows = 6;
+const unsigned int sgemm_Col_NN_B1_MX096_NX096_KX16_microTileNumCols = 6;
+const unsigned int sgemm_Col_NN_B1_MX096_NX096_KX16_unroll = 16;
+
+const char * const sgemm_Col_NN_B1_MX096_NX096_KX16_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rA[0][4] = lA[offA + 64]; \
+ rA[0][5] = lA[offA + 80]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ rB[0][4] = lB[offB + 64]; \
+ rB[0][5] = lB[offB + 80]; \
+ offA += 97; \
+ offB += 97; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]); \
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]); \
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]); \
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]); \
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]); \
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]); \
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]); \
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]); \
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]); \
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]); \
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]); \
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]); \
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]); \
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]); \
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]); \
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]); \
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NN_B1_MX096_NX096_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[6][6] = {(float)0};
+ float rA[1][6];
+ float rB[1][6];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1552];
+ __local float lB[1552];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += gidx*96+ idx + idy*lda;
+ B += gidy*96*ldb+ idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do {
+ __local float* plA = lA + idy*97+idx;
+ __local float* plB = lB + idx*97+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+ plB[64] = B[64*ldb];
+ plB[80] = B[80*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+ plA[64] = A[64+0*lda];
+ plA[80] = A[80+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+
+ A += lda<<4;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*96+idx;
+ C+= gidy*96*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[0][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[0][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[1][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[1][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[2][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[2][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[3][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[3][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[4][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[4][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[4][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[4][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[4][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[5][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[5][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[5][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[5][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[5][5] + beta*C[80*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX032_NX032_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX032_NX032_KX16_src.cpp
new file mode 100644
index 0000000..7cbabb6
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX032_NX032_KX16_src.cpp
@@ -0,0 +1,126 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B0_MX032_NX032_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NT_B0_MX032_NX032_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B0_MX032_NX032_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B0_MX032_NX032_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B0_MX032_NX032_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B0_MX032_NX032_KX16_microTileNumRows = 2;
+const unsigned int sgemm_Col_NT_B0_MX032_NX032_KX16_microTileNumCols = 2;
+const unsigned int sgemm_Col_NT_B0_MX032_NX032_KX16_unroll = 16;
+
+const char * const sgemm_Col_NT_B0_MX032_NX032_KX16_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B0_MX032_NX032_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[544];
+ __local float lB[544];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*32+ idxT + idyT*lda;
+ B += gidy*32+ idxT + idyT*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idyT*33+idxT;
+ __local float* plB = lB + idyT*33+idxT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0+0*ldb];
+ plB[16] = B[16+0*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += lda<<4;
+ B += ldb<<4;
+ } while (--block_k > 0);
+
+ C+= gidx*32+idx;
+ C+= gidy*32*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[16*ldc] = alpha*rC[0][1];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[16*ldc] = alpha*rC[1][1];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX064_NX064_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX064_NX064_KX16_src.cpp
new file mode 100644
index 0000000..5bacf7e
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX064_NX064_KX16_src.cpp
@@ -0,0 +1,165 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B0_MX064_NX064_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NT_B0_MX064_NX064_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B0_MX064_NX064_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B0_MX064_NX064_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B0_MX064_NX064_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B0_MX064_NX064_KX16_microTileNumRows = 4;
+const unsigned int sgemm_Col_NT_B0_MX064_NX064_KX16_microTileNumCols = 4;
+const unsigned int sgemm_Col_NT_B0_MX064_NX064_KX16_unroll = 16;
+
+const char * const sgemm_Col_NT_B0_MX064_NX064_KX16_src = STRINGIFY(
+
+#define M4x4 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ offA += 65; \
+ offB += 65; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+
+__kernel void sgemm_Col_NT_B0_MX064_NX064_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[4][4] = {(float)0};
+ float rA[1][4];
+ float rB[1][4];
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1056];
+ __local float lB[1056];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*64+ idxT + idyT*lda;
+ B += gidy*64+ idxT + idyT*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idyT*65+idxT;
+ __local float* plB = lB + idyT*65+idxT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0+0*ldb];
+ plB[16] = B[16+0*ldb];
+ plB[32] = B[32+0*ldb];
+ plB[48] = B[48+0*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+
+ A += lda<<4;
+ B += ldb<<4;
+ } while (--block_k > 0);
+
+ C+= gidx*64+idx;
+ C+= gidy*64*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[16*ldc] = alpha*rC[0][1];
+ C[32*ldc] = alpha*rC[0][2];
+ C[48*ldc] = alpha*rC[0][3];
+
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[16*ldc] = alpha*rC[1][1];
+ C[32*ldc] = alpha*rC[1][2];
+ C[48*ldc] = alpha*rC[1][3];
+
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] ;
+ C[16*ldc] = alpha*rC[2][1];
+ C[32*ldc] = alpha*rC[2][2];
+ C[48*ldc] = alpha*rC[2][3];
+
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] ;
+ C[16*ldc] = alpha*rC[3][1];
+ C[32*ldc] = alpha*rC[3][2];
+ C[48*ldc] = alpha*rC[3][3];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX096_NX096_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX096_NX096_KX16_src.cpp
new file mode 100644
index 0000000..6a4a786
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B0_MX096_NX096_KX16_src.cpp
@@ -0,0 +1,210 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B0_MX096_NX096_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NT_B0_MX096_NX096_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B0_MX096_NX096_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B0_MX096_NX096_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B0_MX096_NX096_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B0_MX096_NX096_KX16_microTileNumRows = 6;
+const unsigned int sgemm_Col_NT_B0_MX096_NX096_KX16_microTileNumCols = 6;
+const unsigned int sgemm_Col_NT_B0_MX096_NX096_KX16_unroll = 16;
+
+const char * const sgemm_Col_NT_B0_MX096_NX096_KX16_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rA[0][4] = lA[offA + 64]; \
+ rA[0][5] = lA[offA + 80]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ rB[0][4] = lB[offB + 64]; \
+ rB[0][5] = lB[offB + 80]; \
+ offA += 97; \
+ offB += 97; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]); \
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]); \
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]); \
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]); \
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]); \
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]); \
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]); \
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]); \
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]); \
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]); \
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]); \
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]); \
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]); \
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]); \
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]); \
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]); \
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+
+__kernel void sgemm_Col_NT_B0_MX096_NX096_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[6][6] = {(float)0};
+ float rA[1][6];
+ float rB[1][6];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1552];
+ __local float lB[1552];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += gidx*96+ idx + idy*lda;
+ B += gidy*96+ idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy*97+idx;
+ __local float* plB = lB + idy*97+idx;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0+0*ldb];
+ plB[16] = B[16+0*ldb];
+ plB[32] = B[32+0*ldb];
+ plB[48] = B[48+0*ldb];
+ plB[64] = B[64+0*ldb];
+ plB[80] = B[80+0*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+ plA[64] = A[64+0*lda];
+ plA[80] = A[80+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+
+ A += lda<<4;
+ B += ldb<<4;
+ } while (--block_k > 0);
+
+ C+= gidx*96+idx;
+ C+= gidy*96*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[0][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[0][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[1][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[1][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[2][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[2][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[3][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[3][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[4][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[4][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[4][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[4][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[4][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[5][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[5][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[5][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[5][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[5][5] + beta*C[80*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src.cpp
new file mode 100644
index 0000000..971786c
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src.cpp
@@ -0,0 +1,148 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX032_NX032_KX16_BRANCH_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX032_NX032_KX16_BRANCH_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src (if exists) overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_microTileNumRows = 2;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_microTileNumCols = 2;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_unroll = 16;
+
+#ifndef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+unsigned char *sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_bin = 0;
+size_t sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_binSize = 0;
+#endif
+
+const char * const sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[528];//16*32+16
+ __local float lB[528];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ int CurrentOffSetA = gidx*32+ idx;
+ int CurrentOffSetB = gidy*32+ idx;
+
+ A += gidx*32+ idx + idy*lda;
+ B += gidy*32+ idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy*33+idx;
+ __local float* plB = lB + idy*33+idx;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ plB[0] = CurrentOffSetB>=N?0.0:B[0];
+ plB[16] = CurrentOffSetB+16>=N?0.0:B[16];
+
+ plA[0] = CurrentOffSetA>=M?0.0:A[0];
+ plA[16] = CurrentOffSetA+16>=M?0.0:A[16];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += lda<<4;
+ B += ldb<<4;
+ } while (--block_k > 0);
+
+
+ int offset_x = gidx*32+idx;
+ int offset_y = gidy*32+ idy;
+
+ if(offset_x>=M || offset_y>=N )
+ return;
+
+ C+=offset_x+offset_y*ldc;
+
+ int i = 0;
+ do
+ {
+ C[0 ] = mad(alpha, rC[i][0], beta*C[0]);
+ if(offset_y+16<N)
+ C[16*ldc] = mad(alpha, rC[i][1], beta*C[16*ldc]);
+
+ C+=16;
+ offset_x+=16;
+ if(offset_x>=M )
+ return;
+
+
+ }
+ while (++i < 2);
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src.cpp
new file mode 100644
index 0000000..be5b219
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src.cpp
@@ -0,0 +1,158 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ * below kernels work with an assumption: after the main matrix being computed by kernels with 64x64 micro tile size, the boundary are of size 32.
+ * Thus, M and N are of mod32 and not necessarily of mod64.
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX032_NX032_KX16_SINGLE_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX032_NX032_KX16_SINGLE_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src (if exists) overriden by user.")
+#include "UserGemmKernelSourceIncludes.h"
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_microTileNumRows = 2;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_microTileNumCols = 2;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_unroll = 16;
+
+#ifndef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+unsigned char *sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_bin = 0;
+size_t sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_binSize = 0;
+#endif
+
+const char * const sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = { (float)0 };
+ float rA[1][2];
+ float rB[1][2];
+
+
+ A += offsetA;
+ B += offsetB;
+ C += offsetC;
+
+ __local float lA[528];
+ __local float lB[528];
+
+ uint gidx = M / 64;//get_group_id(0);
+ uint gidy = N / 64;//get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ int CurrentOffSetA = gidx * 64 + idx;
+ int CurrentOffSetB = gidy * 64 + idx;
+
+ A += gidx * 64 + idx + idy*lda;
+ B += gidy * 64 + idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy * 33 + idx;
+ __local float* plB = lB + idy * 33 + idx;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ //plB[0] = CurrentOffSetB>=N?0.0:B[0];
+ //plB[16] = CurrentOffSetB+16>=N?0.0:B[16];
+ //plB[32] = CurrentOffSetB+32>=N?0.0:B[32];
+ //plB[48] = CurrentOffSetB+48>=N?0.0:B[48];
+ plB[0] = B[0];
+ plB[16] = B[16];
+
+ //plA[0] = CurrentOffSetA>=M?0.0:A[0];
+ //plA[16] = CurrentOffSetA+16>=M?0.0:A[16];
+ //plA[32] = CurrentOffSetA+32>=M?0.0:A[32];
+ //plA[48] = CurrentOffSetA+48>=M?0.0:A[48];
+ plA[0] = A[0];
+ plA[16] = A[16];
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += lda << 4;
+ B += ldb << 4;
+ } while (--block_k > 0);
+
+
+ int offset_x = gidx * 64 + idx;
+ int offset_y = gidy * 64 + idy;
+
+ //if(offset_x>=M || offset_y>=N )
+ // return;
+
+ C += offset_x + offset_y*ldc;
+
+ int i = 0;
+ do
+ {
+ C[0] = mad(alpha, rC[i][0], beta*C[0]);
+ C[16 * ldc] = mad(alpha, rC[i][1], beta*C[16 * ldc]);
+
+
+ C += 16;
+ offset_x += 16;
+ //if(offset_x>=M )
+ // return;
+
+
+ } while (++i < 2);
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_src.cpp
new file mode 100644
index 0000000..4f3b3d1
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX032_KX16_src.cpp
@@ -0,0 +1,126 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX032_NX032_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX032_NX032_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX032_NX032_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_microTileNumRows = 2;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_microTileNumCols = 2;
+const unsigned int sgemm_Col_NT_B1_MX032_NX032_KX16_unroll = 16;
+
+const char * const sgemm_Col_NT_B1_MX032_NX032_KX16_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX032_NX032_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[544];
+ __local float lB[544];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*32+ idxT + idyT*lda;
+ B += gidy*32+ idxT + idyT*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idyT*33+idxT;
+ __local float* plB = lB + idyT*33+idxT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0+0*ldb];
+ plB[16] = B[16+0*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += lda<<4;
+ B += ldb<<4;
+ } while (--block_k > 0);
+
+ C+= gidx*32+idx;
+ C+= gidy*32*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src.cpp
new file mode 100644
index 0000000..5c41406
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src.cpp
@@ -0,0 +1,161 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ * below kernels work with an assumption: after the main matrix being computed by kernels with 64x64 micro tile size, the boundary are of size 32.
+ * Thus, M and N are of mod32 and not necessarily of mod64.
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX032_NX064_KX16_ROW_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX032_NX064_KX16_ROW_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src (if exists) overriden by user.")
+
+#include "UserGemmKernelSourceIncludes.h"
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_microTileNumRows = 2;
+const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_microTileNumCols = 4;
+const unsigned int sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_unroll = 16;
+
+//if precompiled is enabled. All hand tuned kerenls should be precompiled.
+#ifndef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+unsigned char *sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_bin = 0;
+size_t sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_binSize = 0;
+#endif
+
+const char * const sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src = STRINGIFY(
+
+#define M2x4 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ offA += 33; \
+ offB += 65; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX032_NX064_KX16_ROW (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][4] = { (float)0 };
+ float rA[1][2];
+ float rB[1][4];
+
+
+ A += offsetA;
+ B += offsetB;
+ C += offsetC;
+
+ __local float lA[528];//16*32+16
+ __local float lB[1040];//16*64+16
+
+ uint gidx = M / 64;//get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+
+ int CurrentOffSetA = gidx * 64 + idx;
+
+ A += gidx * 64 + idx + idy*lda;
+ B += gidy * 64 + idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy * 33 + idx;
+ __local float* plB = lB + idy * 65 + idx;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ plB[0] = B[0 + 0 * ldb];
+ plB[16] = B[16 + 0 * ldb];
+ plB[32] = B[32 + 0 * ldb];
+ plB[48] = B[48 + 0 * ldb];
+
+ //plA[0] = CurrentOffSetA>=M?0.0:A[0];
+ //plA[16] = CurrentOffSetA+16>=M?0.0:A[16];
+ //plA[32] = CurrentOffSetA+32>=M?0.0:A[32];
+ //plA[48] = CurrentOffSetA+48>=M?0.0:A[48];
+ plA[0] = A[0];
+ plA[16] = A[16];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+ M2x4
+
+ A += lda << 4;
+ B += ldb << 4;
+ } while (--block_k > 0);
+
+
+ int offset_x = gidx * 64 + idx;
+ int offset_y = gidy * 64 + idy;
+
+ //if(offset_x>=M )
+ // return;
+
+ C += offset_x + offset_y*ldc;
+
+ int i = 0;
+ do
+ {
+ C[0] = mad(alpha, rC[i][0], beta*C[0]);
+ C[16 * ldc] = mad(alpha, rC[i][1], beta*C[16 * ldc]);
+ C[32 * ldc] = mad(alpha, rC[i][2], beta*C[32 * ldc]);
+ C[48 * ldc] = mad(alpha, rC[i][3], beta*C[48 * ldc]);
+ C += 16;
+ offset_x += 16;
+ //if(offset_x>=M )
+ // return;
+ } while (++i < 2);
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX032_KX16_COL_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX032_KX16_COL_src.cpp
new file mode 100644
index 0000000..2c9e9ff
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX032_KX16_COL_src.cpp
@@ -0,0 +1,157 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ * below kernels work with an assumption: after the main matrix being computed by kernels with 64x64 micro tile size, the boundary are of size 32.
+ * Thus, M and N are of mod32 and not necessarily of mod64.
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX064_NX032_KX16_COLUMN_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX064_NX032_KX16_COLUMN_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_src (if exists) overriden by user.")
+#include "UserGemmKernelSourceIncludes.h"
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_microTileNumRows = 4;
+const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_microTileNumCols = 2;
+const unsigned int sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_unroll = 16;
+
+//if precompiled is enabled. All hand tuned kerenls should be precompiled.
+#ifndef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+unsigned char *sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_bin = 0;
+size_t sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_binSize = 0;
+#endif
+
+const char * const sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_src = STRINGIFY(
+
+#define M4x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 65; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[4][2] = { (float)0 };
+ float rA[1][4];
+ float rB[1][2];
+
+
+ A += offsetA;
+ B += offsetB;
+ C += offsetC;
+
+ __local float lA[1040];//16*64+16
+ __local float lB[528];//16*32+16
+
+ uint gidx = get_group_id(0);
+ uint gidy = N / 64;//get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ int CurrentOffSetB = gidy * 64 + idx;
+
+ A += gidx * 64 + idx + idy*lda;
+ B += gidy * 64 + idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy * 65 + idx;
+ __local float* plB = lB + idy * 33 + idx;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ //plB[0] = CurrentOffSetB>=N?0.0:B[0];
+ //plB[16] = CurrentOffSetB+16>=N?0.0:B[16];
+ //plB[32] = CurrentOffSetB+32>=N?0.0:B[32];
+ //plB[48] = CurrentOffSetB+48>=N?0.0:B[48];
+ plB[0] = B[0];
+ plB[16] = B[16];
+
+ plA[0] = A[0];
+ plA[16] = A[16];
+ plA[32] = A[32];
+ plA[48] = A[48];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+ M4x2
+
+ A += lda << 4;
+ B += ldb << 4;
+ } while (--block_k > 0);
+
+
+ int offset_x = gidx * 64 + idx;
+ int offset_y = gidy * 64 + idy;
+
+ //if(offset_y>=N )
+ // return;
+
+ C += offset_x + offset_y*ldc;
+
+ int i = 0;
+ do
+ {
+ C[0] = mad(alpha, rC[i][0], beta*C[0]);
+ C[16 * ldc] = mad(alpha, rC[i][1], beta*C[16 * ldc]);
+
+ C += 16;
+
+ } while (++i < 4);
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX064_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX064_KX16_src.cpp
new file mode 100644
index 0000000..77db018
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX064_NX064_KX16_src.cpp
@@ -0,0 +1,160 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX064_NX064_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX064_NX064_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX064_NX064_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX064_NX064_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX064_NX064_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX064_NX064_KX16_microTileNumRows = 4;
+const unsigned int sgemm_Col_NT_B1_MX064_NX064_KX16_microTileNumCols = 4;
+const unsigned int sgemm_Col_NT_B1_MX064_NX064_KX16_unroll = 16;
+
+const char * const sgemm_Col_NT_B1_MX064_NX064_KX16_src = STRINGIFY(
+
+#define M4x4 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ offA += 65; \
+ offB += 65; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX064_NX064_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[4][4] = {(float)0};
+ float rA[1][4];
+ float rB[1][4];
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1040];
+ __local float lB[1040];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*64+ idxT + idyT*lda;
+ B += gidy*64+ idxT + idyT*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idyT*65+idxT;
+ __local float* plB = lB + idyT*65+idxT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0+0*ldb];
+ plB[16] = B[16+0*ldb];
+ plB[32] = B[32+0*ldb];
+ plB[48] = B[48+0*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+
+ A += lda<<4;
+ B += ldb<<4;
+ } while (--block_k > 0);
+
+ C+= gidx*64+idx;
+ C+= gidy*64*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX096_NX096_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX096_NX096_KX16_src.cpp
new file mode 100644
index 0000000..7ac3d69
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX096_NX096_KX16_src.cpp
@@ -0,0 +1,208 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX096_NX096_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX096_NX096_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX096_NX096_KX16_src overriden by user.")
+
+#include "UserGemmKernelSourceIncludes.h"
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX096_NX096_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX096_NX096_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX096_NX096_KX16_microTileNumRows = 6;
+const unsigned int sgemm_Col_NT_B1_MX096_NX096_KX16_microTileNumCols = 6;
+const unsigned int sgemm_Col_NT_B1_MX096_NX096_KX16_unroll = 16;
+
+const char * const sgemm_Col_NT_B1_MX096_NX096_KX16_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rA[0][4] = lA[offA + 64]; \
+ rA[0][5] = lA[offA + 80]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ rB[0][4] = lB[offB + 64]; \
+ rB[0][5] = lB[offB + 80]; \
+ offA += 97; \
+ offB += 97; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]); \
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]); \
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]); \
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]); \
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]); \
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]); \
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]); \
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]); \
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]); \
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]); \
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]); \
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]); \
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]); \
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]); \
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]); \
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]); \
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX096_NX096_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[6][6] = {(float)0};
+ float rA[1][6];
+ float rB[1][6];
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1552];
+ __local float lB[1552];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += gidx*96+ idx + idy*lda;
+ B += gidy*96+ idx + idy*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idy*97+idx;
+ __local float* plB = lB + idy*97+idx;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0+0*ldb];
+ plB[16] = B[16+0*ldb];
+ plB[32] = B[32+0*ldb];
+ plB[48] = B[48+0*ldb];
+ plB[64] = B[64+0*ldb];
+ plB[80] = B[80+0*ldb];
+
+ plA[0] = A[0+0*lda];
+ plA[16] = A[16+0*lda];
+ plA[32] = A[32+0*lda];
+ plA[48] = A[48+0*lda];
+ plA[64] = A[64+0*lda];
+ plA[80] = A[80+0*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+
+ A += lda<<4;
+ B += ldb<<4;
+ } while (--block_k > 0);
+
+ C+= gidx*96+idx;
+ C+= gidy*96*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[0][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[0][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[1][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[1][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[2][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[2][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[3][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[3][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[4][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[4][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[4][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[4][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[4][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[5][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[5][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[5][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[5][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[5][5] + beta*C[80*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX128_NX128_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX128_NX128_KX16_src.cpp
new file mode 100644
index 0000000..4c5ceb4
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_NT_B1_MX128_NX128_KX16_src.cpp
@@ -0,0 +1,290 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_NT_B1_MX128_NX128_KX16_SRC_H
+#define KERNEL_SGEMM_COL_NT_B1_MX128_NX128_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_NT_B1_MX128_NX128_KX16_src (if exists) overriden by user.")
+
+#include "UserGemmKernelSourceIncludes.h"
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_microTileNumRows = 8;
+const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_microTileNumCols = 8;
+const unsigned int sgemm_Col_NT_B1_MX128_NX128_KX16_unroll = 16;
+
+//if precompiled is enabled. All hand tuned kerenls should be precompiled.
+#ifndef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+unsigned char *sgemm_Col_NT_B1_MX128_NX128_KX16_bin = 0;
+size_t sgemm_Col_NT_B1_MX128_NX128_KX16_binSize = 0;
+#endif
+
+const char * const sgemm_Col_NT_B1_MX128_NX128_KX16_src = STRINGIFY(
+
+#define M8x8 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rA[0][4] = lA[offA + 64]; \
+ rA[0][5] = lA[offA + 80]; \
+ rA[0][6] = lA[offA + 96]; \
+ rA[0][7] = lA[offA + 112]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ rB[0][4] = lB[offB + 64]; \
+ rB[0][5] = lB[offB + 80]; \
+ rB[0][6] = lB[offB + 96]; \
+ rB[0][7] = lB[offB + 112]; \
+ offA += 129; \
+ offB += 129; \
+ rC[0][0] = mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0] = mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0] = mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0] = mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[4][0] = mad(rA[0][4],rB[0][0],rC[4][0]); \
+ rC[5][0] = mad(rA[0][5],rB[0][0],rC[5][0]); \
+ rC[6][0] = mad(rA[0][6],rB[0][0],rC[6][0]); \
+ rC[7][0] = mad(rA[0][7],rB[0][0],rC[7][0]); \
+ rC[0][1] = mad(rA[0][0], rB[0][1], rC[0][1]); \
+ rC[1][1] = mad(rA[0][1], rB[0][1], rC[1][1]); \
+ rC[2][1] = mad(rA[0][2], rB[0][1], rC[2][1]); \
+ rC[3][1] = mad(rA[0][3], rB[0][1], rC[3][1]); \
+ rC[4][1] = mad(rA[0][4], rB[0][1], rC[4][1]); \
+ rC[5][1] = mad(rA[0][5], rB[0][1], rC[5][1]); \
+ rC[6][1] = mad(rA[0][6], rB[0][1], rC[6][1]); \
+ rC[7][1] = mad(rA[0][7], rB[0][1], rC[7][1]); \
+ rC[0][2] = mad(rA[0][0], rB[0][2], rC[0][2]); \
+ rC[1][2] = mad(rA[0][1], rB[0][2], rC[1][2]); \
+ rC[2][2] = mad(rA[0][2], rB[0][2], rC[2][2]); \
+ rC[3][2] = mad(rA[0][3], rB[0][2], rC[3][2]); \
+ rC[4][2] = mad(rA[0][4], rB[0][2], rC[4][2]); \
+ rC[5][2] = mad(rA[0][5], rB[0][2], rC[5][2]); \
+ rC[6][2] = mad(rA[0][6], rB[0][2], rC[6][2]); \
+ rC[7][2] = mad(rA[0][7], rB[0][2], rC[7][2]); \
+ rC[0][3] = mad(rA[0][0], rB[0][3], rC[0][3]); \
+ rC[1][3] = mad(rA[0][1], rB[0][3], rC[1][3]); \
+ rC[2][3] = mad(rA[0][2], rB[0][3], rC[2][3]); \
+ rC[3][3] = mad(rA[0][3], rB[0][3], rC[3][3]); \
+ rC[4][3] = mad(rA[0][4], rB[0][3], rC[4][3]); \
+ rC[5][3] = mad(rA[0][5], rB[0][3], rC[5][3]); \
+ rC[6][3] = mad(rA[0][6], rB[0][3], rC[6][3]); \
+ rC[7][3] = mad(rA[0][7], rB[0][3], rC[7][3]); \
+ rC[0][4] = mad(rA[0][0], rB[0][4], rC[0][4]); \
+ rC[1][4] = mad(rA[0][1], rB[0][4], rC[1][4]); \
+ rC[2][4] = mad(rA[0][2], rB[0][4], rC[2][4]); \
+ rC[3][4] = mad(rA[0][3], rB[0][4], rC[3][4]); \
+ rC[4][4] = mad(rA[0][4], rB[0][4], rC[4][4]); \
+ rC[5][4] = mad(rA[0][5], rB[0][4], rC[5][4]); \
+ rC[6][4] = mad(rA[0][6], rB[0][4], rC[6][4]); \
+ rC[7][4] = mad(rA[0][7], rB[0][4], rC[7][4]); \
+ rC[0][5] = mad(rA[0][0], rB[0][5], rC[0][5]); \
+ rC[1][5] = mad(rA[0][1], rB[0][5], rC[1][5]); \
+ rC[2][5] = mad(rA[0][2], rB[0][5], rC[2][5]); \
+ rC[3][5] = mad(rA[0][3], rB[0][5], rC[3][5]); \
+ rC[4][5] = mad(rA[0][4], rB[0][5], rC[4][5]); \
+ rC[5][5] = mad(rA[0][5], rB[0][5], rC[5][5]); \
+ rC[6][5] = mad(rA[0][6], rB[0][5], rC[6][5]); \
+ rC[7][5] = mad(rA[0][7], rB[0][5], rC[7][5]); \
+ rC[0][6] = mad(rA[0][0], rB[0][6], rC[0][6]); \
+ rC[1][6] = mad(rA[0][1], rB[0][6], rC[1][6]); \
+ rC[2][6] = mad(rA[0][2], rB[0][6], rC[2][6]); \
+ rC[3][6] = mad(rA[0][3], rB[0][6], rC[3][6]); \
+ rC[4][6] = mad(rA[0][4], rB[0][6], rC[4][6]); \
+ rC[5][6] = mad(rA[0][5], rB[0][6], rC[5][6]); \
+ rC[6][6] = mad(rA[0][6], rB[0][6], rC[6][6]); \
+ rC[7][6] = mad(rA[0][7], rB[0][6], rC[7][6]); \
+ rC[0][7] = mad(rA[0][0], rB[0][7], rC[0][7]); \
+ rC[1][7] = mad(rA[0][1], rB[0][7], rC[1][7]); \
+ rC[2][7] = mad(rA[0][2], rB[0][7], rC[2][7]); \
+ rC[3][7] = mad(rA[0][3], rB[0][7], rC[3][7]); \
+ rC[4][7] = mad(rA[0][4], rB[0][7], rC[4][7]); \
+ rC[5][7] = mad(rA[0][5], rB[0][7], rC[5][7]); \
+ rC[6][7] = mad(rA[0][6], rB[0][7], rC[6][7]); \
+ rC[7][7] = mad(rA[0][7], rB[0][7], rC[7][7]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_NT_B1_MX128_NX128_KX16(
+__global float const * restrict A,
+__global float const * restrict B,
+__global float * C,
+float const alpha,
+float const beta,
+uint const M,
+uint const N,
+uint const K,
+uint lda,
+uint ldb,
+uint ldc,
+uint offsetA,
+uint offsetB,
+uint offsetC)
+{
+ float rC[8][8] = { (float)0 };
+ float rA[1][8];
+ float rB[1][8];
+
+ A += offsetA;
+ B += offsetB;
+ C += offsetC;
+
+ __local float lA[2064];
+ __local float lB[2064];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16 * idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx * 128 + idxT + idyT*lda;
+ B += gidy * 128 + idxT + idyT*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ // for(unsigned int block_k=0 ; block_k< K ; block_k+=16)
+ //{
+ __local float* plA = lA + idyT * 129 + idxT;
+ __local float* plB = lB + idyT * 129 + idxT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0 + 0 * ldb];
+ plB[16] = B[16 + 0 * ldb];
+ plB[32] = B[32 + 0 * ldb];
+ plB[48] = B[48 + 0 * ldb];
+ plB[64] = B[64 + 0 * ldb];
+ plB[80] = B[80 + 0 * ldb];
+ plB[96] = B[96 + 0 * ldb];
+ plB[112] = B[112 + 0 * ldb];
+
+ plA[0] = A[0 + 0 * lda];
+ plA[16] = A[16 + 0 * lda];
+ plA[32] = A[32 + 0 * lda];
+ plA[48] = A[48 + 0 * lda];
+ plA[64] = A[64 + 0 * lda];
+ plA[80] = A[80 + 0 * lda];
+ plA[96] = A[96 + 0 * lda];
+ plA[112] = A[112 + 0 * lda];
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ // #pragma unroll 1
+ // for(unsigned int k = 0 ; k < 16; k+=1){
+ // }
+
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+ M8x8
+
+ A += lda << 4;
+ B += ldb << 4;
+ //}
+ } while (--block_k > 0);
+
+ C += gidx * 128 + idx;
+ C += gidy * 128 * ldc;
+ C += idy*ldc;
+
+ C[0 * ldc] = alpha*rC[0][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[0][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[0][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[0][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[0][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[0][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[0][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[0][7] + beta*C[112 * ldc];
+ C += 16;
+ C[0 * ldc] = alpha*rC[1][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[1][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[1][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[1][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[1][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[1][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[1][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[1][7] + beta*C[112 * ldc];
+ C += 16;
+ C[0 * ldc] = alpha*rC[2][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[2][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[2][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[2][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[2][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[2][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[2][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[2][7] + beta*C[112 * ldc];
+ C += 16;
+ C[0 * ldc] = alpha*rC[3][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[3][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[3][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[3][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[3][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[3][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[3][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[3][7] + beta*C[112 * ldc];
+ C += 16;
+ C[0 * ldc] = alpha*rC[4][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[4][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[4][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[4][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[4][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[4][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[4][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[4][7] + beta*C[112 * ldc];
+ C += 16;
+ C[0 * ldc] = alpha*rC[5][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[5][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[5][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[5][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[5][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[5][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[5][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[5][7] + beta*C[112 * ldc];
+ C += 16;
+ C[0 * ldc] = alpha*rC[6][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[6][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[6][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[6][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[6][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[6][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[6][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[6][7] + beta*C[112 * ldc];
+ C += 16;
+ C[0 * ldc] = alpha*rC[7][0] + beta*C[0 * ldc];
+ C[16 * ldc] = alpha*rC[7][1] + beta*C[16 * ldc];
+ C[32 * ldc] = alpha*rC[7][2] + beta*C[32 * ldc];
+ C[48 * ldc] = alpha*rC[7][3] + beta*C[48 * ldc];
+ C[64 * ldc] = alpha*rC[7][4] + beta*C[64 * ldc];
+ C[80 * ldc] = alpha*rC[7][5] + beta*C[80 * ldc];
+ C[96 * ldc] = alpha*rC[7][6] + beta*C[96 * ldc];
+ C[112 * ldc] = alpha*rC[7][7] + beta*C[112 * ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX032_NX032_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX032_NX032_KX16_src.cpp
new file mode 100644
index 0000000..c61bcb7
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX032_NX032_KX16_src.cpp
@@ -0,0 +1,128 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_TN_B0_MX032_NX032_KX16_SRC_H
+#define KERNEL_SGEMM_COL_TN_B0_MX032_NX032_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_TN_B0_MX032_NX032_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_TN_B0_MX032_NX032_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_TN_B0_MX032_NX032_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_TN_B0_MX032_NX032_KX16_microTileNumRows = 2;
+const unsigned int sgemm_Col_TN_B0_MX032_NX032_KX16_microTileNumCols = 2;
+const unsigned int sgemm_Col_TN_B0_MX032_NX032_KX16_unroll = 16;
+
+const char * const sgemm_Col_TN_B0_MX032_NX032_KX16_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_TN_B0_MX032_NX032_KX16_src (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[544];
+ __local float lB[544];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += (gidx*32+idyT)*lda+ idxT ;
+ B += (gidy*32+idyT)*ldb + idxT;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idxT*33+idyT;
+ __local float* plB = lB + idxT*33+idyT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+
+ plA[0] = A[0];
+ plA[16] = A[16*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += 16;
+ B += 16;
+
+ } while (--block_k > 0);
+
+ C+= gidx*32+idx;
+ C+= gidy*32*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[16*ldc] = alpha*rC[0][1];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[16*ldc] = alpha*rC[1][1];
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX064_NX064_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX064_NX064_KX16_src.cpp
new file mode 100644
index 0000000..669ec0d
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX064_NX064_KX16_src.cpp
@@ -0,0 +1,165 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_TN_B0_MX064_NX064_KX16_SRC_H
+#define KERNEL_SGEMM_COL_TN_B0_MX064_NX064_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_TN_B0_MX064_NX064_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_TN_B0_MX064_NX064_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_TN_B0_MX064_NX064_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_TN_B0_MX064_NX064_KX16_microTileNumRows = 4;
+const unsigned int sgemm_Col_TN_B0_MX064_NX064_KX16_microTileNumCols = 4;
+const unsigned int sgemm_Col_TN_B0_MX064_NX064_KX16_unroll = 16;
+
+const char * const sgemm_Col_TN_B0_MX064_NX064_KX16_src = STRINGIFY(
+
+#define M4x4 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ offA += 65; \
+ offB += 65; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_TN_B0_MX064_NX064_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[4][4] = {(float)0};
+ float rA[1][4];
+ float rB[1][4];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1056];
+ __local float lB[1056];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*64*lda+ idxT + idyT*lda;
+ B += gidy*64*ldb+ idxT + idyT*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idxT*65+idyT;
+ __local float* plB = lB + idxT*65+idyT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+
+ plA[0] = A[0];
+ plA[16] = A[16*lda];
+ plA[32] = A[32*lda];
+ plA[48] = A[48*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+
+ A += 16;
+ B += 16;
+
+ } while (--block_k > 0);
+
+ C+= gidx*64+idx;
+ C+= gidy*64*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] ;
+ C[16*ldc] = alpha*rC[0][1] ;
+ C[32*ldc] = alpha*rC[0][2] ;
+ C[48*ldc] = alpha*rC[0][3] ;
+
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] ;
+ C[16*ldc] = alpha*rC[1][1] ;
+ C[32*ldc] = alpha*rC[1][2] ;
+ C[48*ldc] = alpha*rC[1][3] ;
+
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] ;
+ C[16*ldc] = alpha*rC[2][1] ;
+ C[32*ldc] = alpha*rC[2][2] ;
+ C[48*ldc] = alpha*rC[2][3] ;
+
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] ;
+ C[16*ldc] = alpha*rC[3][1] ;
+ C[32*ldc] = alpha*rC[3][2] ;
+ C[48*ldc] = alpha*rC[3][3] ;
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX096_NX096_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX096_NX096_KX16_src.cpp
new file mode 100644
index 0000000..d216825
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B0_MX096_NX096_KX16_src.cpp
@@ -0,0 +1,209 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_TN_B0_MX096_NX096_KX16_SRC_H
+#define KERNEL_SGEMM_COL_TN_B0_MX096_NX096_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_TN_B0_MX096_NX096_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_TN_B0_MX096_NX096_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_TN_B0_MX096_NX096_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_TN_B0_MX096_NX096_KX16_microTileNumRows = 6;
+const unsigned int sgemm_Col_TN_B0_MX096_NX096_KX16_microTileNumCols = 6;
+const unsigned int sgemm_Col_TN_B0_MX096_NX096_KX16_unroll = 16;
+
+const char * const sgemm_Col_TN_B0_MX096_NX096_KX16_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rA[0][4] = lA[offA + 64]; \
+ rA[0][5] = lA[offA + 80]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ rB[0][4] = lB[offB + 64]; \
+ rB[0][5] = lB[offB + 80]; \
+ offA += 97; \
+ offB += 97; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]); \
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]); \
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]); \
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]); \
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]); \
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]); \
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]); \
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]); \
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]); \
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]); \
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]); \
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]); \
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]); \
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]); \
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]); \
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]); \
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_TN_B0_MX096_NX096_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[6][6] = {(float)0};
+ float rA[1][6];
+ float rB[1][6];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1552];
+ __local float lB[1552];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += (gidx*96+idy)*lda + idx;
+ B += (gidy*96+idy)*ldb + idx;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idx*97+idy;
+ __local float* plB = lB + idx*97+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+ plB[64] = B[64*ldb];
+ plB[80] = B[80*ldb];
+
+ plA[0] = A[0];
+ plA[16] = A[16*lda];
+ plA[32] = A[32*lda];
+ plA[48] = A[48*lda];
+ plA[64] = A[64*lda];
+ plA[80] = A[80*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+
+ A += 16;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*96+idx;
+ C+= gidy*96*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[0][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[0][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[1][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[1][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[2][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[2][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[3][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[3][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[4][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[4][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[4][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[4][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[4][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[5][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[5][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[5][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[5][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[5][5] + beta*C[80*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src.cpp
new file mode 100644
index 0000000..65d1ecb
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src.cpp
@@ -0,0 +1,148 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_TN_B1_MX032_NX032_KX16_BRANCH_SRC_H
+#define KERNEL_SGEMM_COL_TN_B1_MX032_NX032_KX16_BRANCH_SRC_H
+#pragma message("AutoGemm's sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_workGroupNumRows = 16;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_workGroupNumCols = 16;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_microTileNumRows = 2;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_microTileNumCols = 2;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_unroll = 16;
+
+#ifndef AUTOGEMM_USE_PRE_COMPILED_KERNELS
+unsigned char *sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_bin = 0;
+size_t sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_binSize = 0;
+#endif
+
+const char * const sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[528];//16*32+16
+ __local float lB[528];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ int CurrentOffSetA = gidx*32+ idy;
+ int CurrentOffSetB = gidy*32+ idy;
+
+ A += (gidx*32+idy)*lda + idx;
+ B += (gidy*32+idy)*ldb + idx;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idx*33+idy;
+ __local float* plB = lB + idx*33+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ plB[0] = CurrentOffSetB>=N?0.0:B[0];
+ plB[16] = CurrentOffSetB+16>=N?0.0:B[16*ldb];
+
+ plA[0] = CurrentOffSetA>=M?0.0:A[0];
+ plA[16] = CurrentOffSetA+16>=M?0.0:A[16*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += 16;
+ B += 16;
+ } while (--block_k > 0);
+
+
+ int offset_x = gidx*32+idx;
+ int offset_y = gidy*32+ idy;
+
+ if(offset_x>=M || offset_y>=N )
+ return;
+
+ C+=offset_x+offset_y*ldc;
+
+ int i = 0;
+ do
+ {
+ C[0 ] = mad(alpha, rC[i][0], beta*C[0]);
+ if(offset_y+16<N)
+ C[16*ldc] = mad(alpha, rC[i][1], beta*C[16*ldc]);
+
+ C+=16;
+ offset_x+=16;
+ if(offset_x>=M )
+ return;
+
+
+ }
+ while (++i < 2);
+}
+
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_src.cpp
new file mode 100644
index 0000000..24259d7
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX032_NX032_KX16_src.cpp
@@ -0,0 +1,127 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_TN_B1_MX032_NX032_KX16_SRC_H
+#define KERNEL_SGEMM_COL_TN_B1_MX032_NX032_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_TN_B1_MX032_NX032_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_microTileNumRows = 2;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_microTileNumCols = 2;
+const unsigned int sgemm_Col_TN_B1_MX032_NX032_KX16_unroll = 16;
+
+const char * const sgemm_Col_TN_B1_MX032_NX032_KX16_src = STRINGIFY(
+
+#define M2x2 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ offA += 33; \
+ offB += 33; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_TN_B1_MX032_NX032_KX16_src (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[2][2] = {(float)0};
+ float rA[1][2];
+ float rB[1][2];
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[544];
+ __local float lB[544];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += (gidx*32+idyT)*lda+ idxT ;
+ B += (gidy*32+idyT)*ldb + idxT;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+
+ __local float* plA = lA + idxT*33+idyT;
+ __local float* plB = lB + idxT*33+idyT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+
+ plA[0] = A[0];
+ plA[16] = A[16*lda];
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+ M2x2
+
+ A += 16;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*32+idx;
+ C+= gidy*32*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+}
+
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX064_NX064_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX064_NX064_KX16_src.cpp
new file mode 100644
index 0000000..48d565c
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX064_NX064_KX16_src.cpp
@@ -0,0 +1,165 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_TN_B1_MX064_NX064_KX16_SRC_H
+#define KERNEL_SGEMM_COL_TN_B1_MX064_NX064_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_TN_B1_MX064_NX064_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_TN_B1_MX064_NX064_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_TN_B1_MX064_NX064_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_TN_B1_MX064_NX064_KX16_microTileNumRows = 4;
+const unsigned int sgemm_Col_TN_B1_MX064_NX064_KX16_microTileNumCols = 4;
+const unsigned int sgemm_Col_TN_B1_MX064_NX064_KX16_unroll = 16;
+
+const char * const sgemm_Col_TN_B1_MX064_NX064_KX16_src = STRINGIFY(
+
+#define M4x4 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ offA += 65; \
+ offB += 65; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_TN_B1_MX064_NX064_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[4][4] = {(float)0};
+ float rA[1][4];
+ float rB[1][4];
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1056];
+ __local float lB[1056];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ uint idt = 16*idy + idx;
+ uint idxT = idt % 16;
+ uint idyT = idt / 16;
+
+ A += gidx*64*lda+ idxT + idyT*lda;
+ B += gidy*64*ldb+ idxT + idyT*ldb;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idxT*65+idyT;
+ __local float* plB = lB + idxT*65+idyT;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+
+ plA[0] = A[0];
+ plA[16] = A[16*lda];
+ plA[32] = A[32*lda];
+ plA[48] = A[48*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+ M4x4
+
+ A += 16;
+ B += 16;
+
+ } while (--block_k > 0);
+
+ C+= gidx*64+idx;
+ C+= gidy*64*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+}
+);
+#endif
diff --git a/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX096_NX096_KX16_src.cpp b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX096_NX096_KX16_src.cpp
new file mode 100644
index 0000000..bfe73b5
--- /dev/null
+++ b/src/library/blas/AutoGemm/UserGemmKernelSources/sgemm_Col_TN_B1_MX096_NX096_KX16_src.cpp
@@ -0,0 +1,209 @@
+/*******************************************************************************
+ * Hand-tuned kernel
+ ******************************************************************************/
+
+#ifndef KERNEL_SGEMM_COL_TN_B1_MX096_NX096_KX16_SRC_H
+#define KERNEL_SGEMM_COL_TN_B1_MX096_NX096_KX16_SRC_H
+#pragma message("AutoGemm's sgemm_Col_TN_B1_MX096_NX096_KX16_src overriden by user.")
+
+#ifndef STRINGIFY
+#define STRINGIFY(S) STRINGIFY2(S)
+#define STRINGIFY2(S) #S
+#endif
+
+const unsigned int sgemm_Col_TN_B1_MX096_NX096_KX16_workGroupNumRows = 16;
+const unsigned int sgemm_Col_TN_B1_MX096_NX096_KX16_workGroupNumCols = 16;
+const unsigned int sgemm_Col_TN_B1_MX096_NX096_KX16_microTileNumRows = 6;
+const unsigned int sgemm_Col_TN_B1_MX096_NX096_KX16_microTileNumCols = 6;
+const unsigned int sgemm_Col_TN_B1_MX096_NX096_KX16_unroll = 16;
+
+const char * const sgemm_Col_TN_B1_MX096_NX096_KX16_src = STRINGIFY(
+
+#define M6x6 \
+ rA[0][0] = lA[offA + 0]; \
+ rA[0][1] = lA[offA + 16]; \
+ rA[0][2] = lA[offA + 32]; \
+ rA[0][3] = lA[offA + 48]; \
+ rA[0][4] = lA[offA + 64]; \
+ rA[0][5] = lA[offA + 80]; \
+ rB[0][0] = lB[offB + 0]; \
+ rB[0][1] = lB[offB + 16]; \
+ rB[0][2] = lB[offB + 32]; \
+ rB[0][3] = lB[offB + 48]; \
+ rB[0][4] = lB[offB + 64]; \
+ rB[0][5] = lB[offB + 80]; \
+ offA += 97; \
+ offB += 97; \
+ rC[0][0]=mad(rA[0][0],rB[0][0],rC[0][0]); \
+ rC[1][0]=mad(rA[0][1],rB[0][0],rC[1][0]); \
+ rC[2][0]=mad(rA[0][2],rB[0][0],rC[2][0]); \
+ rC[3][0]=mad(rA[0][3],rB[0][0],rC[3][0]); \
+ rC[4][0]=mad(rA[0][4],rB[0][0],rC[4][0]); \
+ rC[5][0]=mad(rA[0][5],rB[0][0],rC[5][0]); \
+ rC[0][1]=mad(rA[0][0],rB[0][1],rC[0][1]); \
+ rC[1][1]=mad(rA[0][1],rB[0][1],rC[1][1]); \
+ rC[2][1]=mad(rA[0][2],rB[0][1],rC[2][1]); \
+ rC[3][1]=mad(rA[0][3],rB[0][1],rC[3][1]); \
+ rC[4][1]=mad(rA[0][4],rB[0][1],rC[4][1]); \
+ rC[5][1]=mad(rA[0][5],rB[0][1],rC[5][1]); \
+ rC[0][2]=mad(rA[0][0],rB[0][2],rC[0][2]); \
+ rC[1][2]=mad(rA[0][1],rB[0][2],rC[1][2]); \
+ rC[2][2]=mad(rA[0][2],rB[0][2],rC[2][2]); \
+ rC[3][2]=mad(rA[0][3],rB[0][2],rC[3][2]); \
+ rC[4][2]=mad(rA[0][4],rB[0][2],rC[4][2]); \
+ rC[5][2]=mad(rA[0][5],rB[0][2],rC[5][2]); \
+ rC[0][3]=mad(rA[0][0],rB[0][3],rC[0][3]); \
+ rC[1][3]=mad(rA[0][1],rB[0][3],rC[1][3]); \
+ rC[2][3]=mad(rA[0][2],rB[0][3],rC[2][3]); \
+ rC[3][3]=mad(rA[0][3],rB[0][3],rC[3][3]); \
+ rC[4][3]=mad(rA[0][4],rB[0][3],rC[4][3]); \
+ rC[5][3]=mad(rA[0][5],rB[0][3],rC[5][3]); \
+ rC[0][4]=mad(rA[0][0],rB[0][4],rC[0][4]); \
+ rC[1][4]=mad(rA[0][1],rB[0][4],rC[1][4]); \
+ rC[2][4]=mad(rA[0][2],rB[0][4],rC[2][4]); \
+ rC[3][4]=mad(rA[0][3],rB[0][4],rC[3][4]); \
+ rC[4][4]=mad(rA[0][4],rB[0][4],rC[4][4]); \
+ rC[5][4]=mad(rA[0][5],rB[0][4],rC[5][4]); \
+ rC[0][5]=mad(rA[0][0],rB[0][5],rC[0][5]); \
+ rC[1][5]=mad(rA[0][1],rB[0][5],rC[1][5]); \
+ rC[2][5]=mad(rA[0][2],rB[0][5],rC[2][5]); \
+ rC[3][5]=mad(rA[0][3],rB[0][5],rC[3][5]); \
+ rC[4][5]=mad(rA[0][4],rB[0][5],rC[4][5]); \
+ rC[5][5]=mad(rA[0][5],rB[0][5],rC[5][5]); \
+ mem_fence(CLK_LOCAL_MEM_FENCE);\n
+
+__attribute__((reqd_work_group_size(16,16,1)))
+__kernel void sgemm_Col_TN_B1_MX096_NX096_KX16 (
+ __global float const * restrict A,
+ __global float const * restrict B,
+ __global float * C,
+ float const alpha,
+ float const beta,
+ uint const M,
+ uint const N,
+ uint const K,
+ uint lda,
+ uint ldb,
+ uint ldc,
+ uint offsetA,
+ uint offsetB,
+ uint offsetC)
+{
+ float rC[6][6] = {(float)0};
+ float rA[1][6];
+ float rB[1][6];
+
+
+
+ A += offsetA;
+ B += offsetB;
+ C+=offsetC;
+
+ __local float lA[1552];
+ __local float lB[1552];
+
+ uint gidx = get_group_id(0);
+ uint gidy = get_group_id(1);
+ uint idx = get_local_id(0);
+ uint idy = get_local_id(1);
+
+ A += (gidx*96+idy)*lda + idx;
+ B += (gidy*96+idy)*ldb + idx;
+
+
+ uint block_k = K >> 4;
+ do
+ {
+ __local float* plA = lA + idx*97+idy;
+ __local float* plB = lB + idx*97+idy;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ plB[0] = B[0];
+ plB[16] = B[16*ldb];
+ plB[32] = B[32*ldb];
+ plB[48] = B[48*ldb];
+ plB[64] = B[64*ldb];
+ plB[80] = B[80*ldb];
+
+ plA[0] = A[0];
+ plA[16] = A[16*lda];
+ plA[32] = A[32*lda];
+ plA[48] = A[48*lda];
+ plA[64] = A[64*lda];
+ plA[80] = A[80*lda];
+
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+ uint offA = idx;
+ uint offB = idy;
+
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+ M6x6
+
+ A += 16;
+ B += 16;
+ } while (--block_k > 0);
+
+ C+= gidx*96+idx;
+ C+= gidy*96*ldc;
+ C+= idy*ldc;
+
+ C[0*ldc] = alpha*rC[0][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[0][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[0][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[0][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[0][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[0][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[1][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[1][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[1][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[1][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[1][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[1][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[2][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[2][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[2][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[2][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[2][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[2][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[3][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[3][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[3][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[3][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[3][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[3][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[4][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[4][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[4][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[4][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[4][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[4][5] + beta*C[80*ldc];
+ C+=16;
+ C[0*ldc] = alpha*rC[5][0] + beta*C[0*ldc];
+ C[16*ldc] = alpha*rC[5][1] + beta*C[16*ldc];
+ C[32*ldc] = alpha*rC[5][2] + beta*C[32*ldc];
+ C[48*ldc] = alpha*rC[5][3] + beta*C[48*ldc];
+ C[64*ldc] = alpha*rC[5][4] + beta*C[64*ldc];
+ C[80*ldc] = alpha*rC[5][5] + beta*C[80*ldc];
+
+}
+);
+#endif
diff --git a/src/library/blas/include/xgemm.h b/src/library/blas/include/xgemm.h
new file mode 100644
index 0000000..42bfaff
--- /dev/null
+++ b/src/library/blas/include/xgemm.h
@@ -0,0 +1,39 @@
+/* ************************************************************************
+* Copyright 2015 Advanced Micro Devices, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+* ************************************************************************/
+
+//some help functions
+
+#ifndef CLBLAS_XGEMM_H
+#define CLBLAS_XGEMM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void makeGemmKernel(
+ cl_kernel *clKernel,
+ cl_command_queue clQueue,
+ const char *kernelSource,
+ const char *sourceBuildOptions,
+ const unsigned char **kernelBinary,
+ size_t *kernelBinarySize,
+ const char *binaryBuildOptions);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/library/blas/specialCases/GemmSpecialCases.cpp b/src/library/blas/specialCases/GemmSpecialCases.cpp
new file mode 100644
index 0000000..4a5fd82
--- /dev/null
+++ b/src/library/blas/specialCases/GemmSpecialCases.cpp
@@ -0,0 +1,829 @@
+/* ************************************************************************
+* Copyright 2015 Advanced Micro Devices, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+* ************************************************************************/
+
+#include "GemmSpecialCases.h"
+#include "UserGemmKernelSources/UserGemmKernelSourceIncludes.h"
+#include "UserGemmKernelSources/UserGemmClKernels.h"
+#include "xgemm.h" //helper functions defined in xgemm.cpp
+#include "AutoGemmIncludes/AutoGemmClKernels.h"
+#include "AutoGemmIncludes/AutoGemmKernelSources.h"
+#include "AutoGemmIncludes/AutoGemmKernelBinaries.h"
+
+/******************************************************************************
+* Check OpenCL Errors
+*****************************************************************************/
+#define CL_CHECK(RET) \
+ if(RET != CL_SUCCESS) { \
+ printf("OpenCL error %i on line %u\n", RET, __LINE__); \
+ assert(false); \
+ }
+
+clblasStatus SGEMM_SPLIT_CALLS(
+ cl_kernel *ClKernel, clblasOrder order,
+ unsigned int tile_size, unsigned int WG_size,
+ unsigned int M_split_factor,
+ unsigned int N_split_factor,
+ unsigned int K_split_factor,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ cl_uint M, cl_uint N, cl_uint K,
+ float alpha,
+ cl_mem A, cl_uint offA, cl_uint lda,
+ cl_mem B, cl_uint offB, cl_uint ldb,
+ float beta,
+ cl_mem C, cl_uint offC, cl_uint ldc,
+ cl_uint numCommandQueues,
+ cl_command_queue *commandQueues,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *events)
+{
+ //for example, when M=N=K=8192
+ //we are gonna call 16 GEMMs
+ //each GEMM has M=N=K=4096
+ //note are direct GEMM call has a 0.7 TFLOPS performance
+
+ // [ A11 | A12 | A13 | A14 ] [ B11 | B12 | B13 | B14 ] [ C11 | C12 ]
+ // A = [ A21 | A22 | A23 | A24 ] B = [ B21 | B22 | B23 | B24 ] C = [ C21 | C22 ]
+
+ // 16 GEMMs are
+ // #01: C11 = a*A11*B11 + b*C11
+ // #02: C11 = a*A12*B12 + 1*C11
+ // #03: C11 = a*A13*B13 + 1*C11
+ // #04: C11 = a*A14*B14 + 1*C11 now we are done with C11
+
+ // #05: C12 = a*A11*B21 + b*C12
+ // #06: C12 = a*A12*B22 + 1*C12
+ // #07: C12 = a*A12*B22 + 1*C12
+ // #08: C12 = a*A12*B22 + 1*C12 now we are done with C12
+
+ // #09: C21 = a*A21*B11 + b*C21
+ // #10: C21 = a*A22*B12 + 1*C21
+ // #11: C21 = a*A23*B13 + 1*C21
+ // #12: C21 = a*A24*B14 + 1*C21 now we are done with C21
+
+ // #13: C22 = a*A21*B21 + b*C22
+ // #14: C22 = a*A22*B22 + 1*C22
+ // #15: C22 = a*A23*B23 + 1*C22
+ // #16: C22 = a*A24*B24 + 1*C22 now we are done with C22
+
+ unsigned int small_M = M / M_split_factor;
+ unsigned int small_N = N / N_split_factor;
+ unsigned int small_K = K / K_split_factor;
+
+ size_t GlobalX = ((small_M - 1) / tile_size + 1) * WG_size;
+ size_t GlobalY = ((small_N - 1) / tile_size + 1) * WG_size;
+ size_t gs[2] = { GlobalX, GlobalY };
+ size_t wgsize[2] = { WG_size, WG_size };
+ cl_int error = 0;
+
+ cl_float betaone = 1;
+
+ error = clSetKernelArg(*ClKernel, 5, sizeof(cl_uint), &small_M);
+ assert(error == CL_SUCCESS);
+ error = clSetKernelArg(*ClKernel, 6, sizeof(cl_uint), &small_N);
+ assert(error == CL_SUCCESS);
+ error = clSetKernelArg(*ClKernel, 7, sizeof(cl_uint), &small_K);
+ assert(error == CL_SUCCESS);
+
+ for (int M_split_index = 0; M_split_index < M_split_factor; M_split_index++)
+ {
+ for (int N_split_index = 0; N_split_index < N_split_factor; N_split_index++)
+ {
+ unsigned int offc_C = ldc*N / N_split_factor * N_split_index + M / M_split_factor * M_split_index + offC;
+ error = clSetKernelArg(*ClKernel, 13, sizeof(cl_uint), &offc_C);
+ assert(error == CL_SUCCESS);
+
+ for (int K_split_index = 0; K_split_index < K_split_factor; K_split_index++)
+ {
+ unsigned int offa_A = (M / M_split_factor * M_split_index) + (lda * K / K_split_factor * K_split_index) + offA;
+ unsigned int offb_B = (N / N_split_factor * N_split_index) + (ldb * K / K_split_factor * K_split_index) + offB;
+ error = clSetKernelArg(*ClKernel, 11, sizeof(cl_uint), &offa_A);
+ assert(error == CL_SUCCESS);
+ error = clSetKernelArg(*ClKernel, 12, sizeof(cl_uint), &offb_B);
+ assert(error == CL_SUCCESS);
+
+ if (K_split_index == 0)
+ {
+ error = clSetKernelArg(*ClKernel, 4, sizeof(cl_float), &(beta));
+ assert(error == CL_SUCCESS);
+
+ if (M_split_index == 0 && N_split_index == 0)
+ {
+ //very first GEMM call
+ if ((M_split_factor == 1) && (N_split_factor == 1) && (K_split_factor == 1))
+ {
+ //also very last GEMM call
+ error = clEnqueueNDRangeKernel(commandQueues[0], *ClKernel, 2, NULL,
+ gs, wgsize, numEventsInWaitList, eventWaitList, &events[0]);
+ assert(error == CL_SUCCESS);
+ }
+ else
+ {
+ error = clEnqueueNDRangeKernel(commandQueues[0], *ClKernel, 2, NULL,
+ gs, wgsize, numEventsInWaitList, eventWaitList, NULL);
+ assert(error == CL_SUCCESS);
+ }
+ }
+ else
+ {
+ error = clEnqueueNDRangeKernel(commandQueues[0], *ClKernel, 2, NULL,
+ gs, wgsize, 0, NULL, NULL);
+ assert(error == CL_SUCCESS);
+ }
+ }
+ else
+ {
+ error = clSetKernelArg(*ClKernel, 4, sizeof(cl_float), &betaone);
+ assert(error == CL_SUCCESS);
+
+ if ((M_split_index == (M_split_factor - 1)) && (N_split_index == (N_split_factor - 1)) && (K_split_index == (K_split_factor - 1)))
+ {
+ //very last GEMM call
+ error = clEnqueueNDRangeKernel(commandQueues[0], *ClKernel, 2, NULL,
+ gs, wgsize, 0, NULL, events);
+ assert(error == CL_SUCCESS);
+ }
+ else
+ {
+ error = clEnqueueNDRangeKernel(commandQueues[0], *ClKernel, 2, NULL,
+ gs, wgsize, 0, NULL, NULL);
+ assert(error == CL_SUCCESS);
+ }
+ }
+ }
+ }
+ }
+
+ return clblasSuccess;
+}
+
+clblasStatus GEMM_mod1024(
+ clblasTranspose transA,
+ clblasTranspose transB,
+ cl_uint M, cl_uint N, cl_uint K,
+ float alpha,
+ cl_mem A, cl_uint offA, cl_uint lda,
+ cl_mem B, cl_uint offB, cl_uint ldb,
+ float beta,
+ cl_mem C, cl_uint offC, cl_uint ldc,
+ cl_uint numCommandQueues,
+ cl_command_queue *commandQueues,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *events,
+ bool &specialCaseHandled)
+{
+ const char *tileKernelSource = NULL;
+ cl_kernel *tileClKernel = NULL;
+ size_t tileKernelBinarySize = 0;
+ cl_int err;
+
+
+ const unsigned char *tileKernelBinary = NULL;
+
+ clblasStatus status;
+
+
+ //split the kernel calls to handle sgemm NT perf drop at big multiples of 1024
+ if ((lda % 1024 == 0) && (ldb % 1024 == 0) && (K > lda / 4))
+ {
+ if ((lda == ldb) && (lda >= 4096) && (lda <= 8192)) // between 4096 and 8192 for now
+ {
+ if (lda != 6144)// 6144 is handled by 96 x 96 kernel
+ {
+ // we are going to call 16 GEMMs with M=M/2, N=N/2, K=K/4
+ // each GEMM requires M%128 == 0, N%128 == 0, K%16 == 0
+ if (M % 256 == 0 && N % 256 == 0 && K % 64 == 0)
+ {
+ if (!((transA == clblasNoTrans) && (transB == clblasTrans)))
+ return clblasNotImplemented;
+
+ specialCaseHandled = true;
+ unsigned int M_split_factor;
+ unsigned int N_split_factor;
+ unsigned int K_split_factor;
+
+ if (lda < 7168)
+ {
+ M_split_factor = 1;
+ N_split_factor = 1;
+ K_split_factor = 1;
+ }
+ else
+ {
+ //7168, 8192
+ M_split_factor = 2;
+ N_split_factor = 2;
+ K_split_factor = 4;
+ }
+
+ tileKernelSource = sgemm_Col_NT_B1_MX128_NX128_KX16_src;
+ tileClKernel = &sgemm_Col_NT_B1_MX128_NX128_KX16_clKernel;
+ tileKernelBinary = sgemm_Col_NT_B1_MX128_NX128_KX16_bin;
+ tileKernelBinarySize = sgemm_Col_NT_B1_MX128_NX128_KX16_binSize;
+
+ makeGemmKernel(tileClKernel, commandQueues[0], tileKernelSource, User_srcBuildOptions, &tileKernelBinary, &tileKernelBinarySize, User_binBuildOptions);
+
+ err = clSetKernelArg(*tileClKernel, 0, sizeof(cl_mem), &A);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 1, sizeof(cl_mem), &B);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 2, sizeof(cl_mem), &C);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 3, sizeof(cl_float), &alpha);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 4, sizeof(cl_float), &beta);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 5, sizeof(cl_uint), &M);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 6, sizeof(cl_uint), &N);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 7, sizeof(cl_uint), &K);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 8, sizeof(cl_uint), &lda);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 9, sizeof(cl_uint), &ldb);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 10, sizeof(cl_uint), &ldc);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 11, sizeof(cl_uint), &offA);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 12, sizeof(cl_uint), &offB);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 13, sizeof(cl_uint), &offC);
+ CL_CHECK(err);
+
+ status = SGEMM_SPLIT_CALLS(
+ tileClKernel, clblasColumnMajor,
+ 128, 16,
+ M_split_factor,
+ N_split_factor, K_split_factor,
+ transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events);
+
+
+ return status;
+ }
+ }
+ else
+ {
+ // lda == ldb == 6144
+ // we are going to call 4 GEMMs each with K = K/4
+ if (M % 96 == 0 && N % 96 == 0 && K % 64 == 0)
+ {
+ if (!((transA == clblasNoTrans) && (transB == clblasTrans)))
+ return clblasNotImplemented;
+
+ specialCaseHandled = true;
+ unsigned int M_split_factor = 1;
+ unsigned int N_split_factor = 1;
+ unsigned int K_split_factor = 4;
+
+
+
+ tileKernelSource = sgemm_Col_NT_B1_MX096_NX096_KX16_src;
+ tileClKernel = &sgemm_Col_NT_B1_MX096_NX096_KX16_clKernel;
+ tileKernelBinary = sgemm_Col_NT_B1_MX096_NX096_KX16_bin;
+ tileKernelBinarySize = sgemm_Col_NT_B1_MX096_NX096_KX16_binSize;
+
+ makeGemmKernel(tileClKernel, commandQueues[0], tileKernelSource, User_srcBuildOptions, &tileKernelBinary, &tileKernelBinarySize, User_binBuildOptions);
+
+ err = clSetKernelArg(*tileClKernel, 0, sizeof(cl_mem), &A);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 1, sizeof(cl_mem), &B);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 2, sizeof(cl_mem), &C);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 3, sizeof(cl_float), &alpha);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 4, sizeof(cl_float), &beta);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 5, sizeof(cl_uint), &M);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 6, sizeof(cl_uint), &N);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 7, sizeof(cl_uint), &K);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 8, sizeof(cl_uint), &lda);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 9, sizeof(cl_uint), &ldb);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 10, sizeof(cl_uint), &ldc);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 11, sizeof(cl_uint), &offA);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 12, sizeof(cl_uint), &offB);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 13, sizeof(cl_uint), &offC);
+ CL_CHECK(err);
+
+
+ status = SGEMM_SPLIT_CALLS(
+ tileClKernel, clblasColumnMajor,
+ 96, 16,
+ M_split_factor,
+ N_split_factor, K_split_factor,
+ transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events);
+
+
+ return status;
+ }
+ }
+ }
+ }
+
+ return clblasNotImplemented;
+}
+
+
+clblasStatus GEMM_SPLIT64_32(
+ clblasTranspose transA,
+ clblasTranspose transB,
+ cl_uint M, cl_uint N, cl_uint K,
+ float alpha,
+ cl_mem A, cl_uint offA, cl_uint lda,
+ cl_mem B, cl_uint offB, cl_uint ldb,
+ float beta,
+ cl_mem C, cl_uint offC, cl_uint ldc,
+ cl_uint numCommandQueues,
+ cl_command_queue *commandQueues,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *events,
+ bool &specialCaseHandled)
+{
+ //all the mod32 sizes that is not mod64 or mod96 ranging from 1184 to 3872
+ //non mod32 cases are not implemented in this approach and are of less interest
+ const char *tileKernelSource = NULL;
+ const char *rowKernelSource = NULL;
+ const char *columnKernelSource = NULL;
+ const char *singleKernelSource = NULL;
+
+ cl_kernel *tileClKernel = NULL;
+ cl_kernel *rowClKernel = NULL;
+ cl_kernel *columnClKernel = NULL;
+ cl_kernel *singleClKernel = NULL;
+
+ const unsigned char *tileKernelBinary = NULL;
+ const unsigned char *rowKernelBinary = NULL;
+ const unsigned char *columnKernelBinary = NULL;
+ const unsigned char *singleKernelBinary = NULL;
+
+ size_t tileKernelBinarySize = 0;
+ size_t rowKernelBinarySize = 0;
+ size_t columnKernelBinarySize = 0;
+ size_t singleKernelBinarySize = 0;
+
+ cl_int err;
+
+ if ((M >= 1184 && N >= 1184) && (M <= 3872 && N <= 3872) && (M % 64 != 0 && N % 64 != 0) && (M % 96 != 0 && N % 96 != 0) && (K % 16 == 0))
+ {
+ if ((M % 32 == 0 && N % 32 == 0) && (transA == clblasNoTrans && transB == clblasTrans))
+ {
+ specialCaseHandled = true;
+ //execute the kernels
+
+ //GlobalX = ((Mvalue - 1) / 64) * 16
+ //GlobalY = ((Nvalue - 1) / 64) * 16
+ size_t GlobalX = ((M - 1) / 64) * 16;
+ size_t GlobalY = ((N - 1) / 64) * 16;
+ size_t gs[2] = { GlobalX, GlobalY };
+ size_t wgsize[2] = { 16, 16 };
+
+ tileKernelSource = sgemm_Col_NT_B1_MX064_NX064_KX16_src;
+ tileClKernel = &sgemm_Col_NT_B1_MX064_NX064_KX16_clKernel;
+ tileKernelBinary = sgemm_Col_NT_B1_MX064_NX064_KX16_bin;
+ tileKernelBinarySize = sgemm_Col_NT_B1_MX064_NX064_KX16_binSize;
+
+ rowKernelSource = sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_src;
+ rowClKernel = &sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_clKernel;
+ rowKernelBinary = sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_bin;
+ rowKernelBinarySize = sgemm_Col_NT_B1_MX032_NX064_KX16_ROW_binSize;
+
+ columnKernelSource = sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_src;
+ columnClKernel = &sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_clKernel;
+ columnKernelBinary = sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_bin;
+ columnKernelBinarySize = sgemm_Col_NT_B1_MX064_NX032_KX16_COLUMN_binSize;
+
+ singleKernelSource = sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_src;
+ singleClKernel = &sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_clKernel;
+ singleKernelBinary = sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_bin;
+ singleKernelBinarySize = sgemm_Col_NT_B1_MX032_NX032_KX16_SINGLE_binSize;
+
+ cl_kernel * Kernels[4] = { tileClKernel, rowClKernel, columnClKernel, singleClKernel };
+
+
+ makeGemmKernel(tileClKernel, commandQueues[0], tileKernelSource, User_srcBuildOptions, &tileKernelBinary, &tileKernelBinarySize, User_binBuildOptions);
+ makeGemmKernel(rowClKernel, commandQueues[0], rowKernelSource, User_srcBuildOptions, &rowKernelBinary, &rowKernelBinarySize, User_binBuildOptions);
+ makeGemmKernel(columnClKernel, commandQueues[0], columnKernelSource, User_srcBuildOptions, &columnKernelBinary, &columnKernelBinarySize, User_binBuildOptions);
+ makeGemmKernel(singleClKernel, commandQueues[0], singleKernelSource, User_srcBuildOptions, &singleKernelBinary, &singleKernelBinarySize, User_binBuildOptions);
+
+ for (int i = 0; i < 4; i++)
+ {
+ err = clSetKernelArg(*Kernels[i], 0, sizeof(cl_mem), &A);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 1, sizeof(cl_mem), &B);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 2, sizeof(cl_mem), &C);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 3, sizeof(cl_float), &alpha);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 4, sizeof(cl_float), &beta);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 5, sizeof(cl_uint), &M);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 6, sizeof(cl_uint), &N);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 7, sizeof(cl_uint), &K);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 8, sizeof(cl_uint), &lda);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 9, sizeof(cl_uint), &ldb);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 10, sizeof(cl_uint), &ldc);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 11, sizeof(cl_uint), &offA);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 12, sizeof(cl_uint), &offB);
+ CL_CHECK(err);
+ err = clSetKernelArg(*Kernels[i], 13, sizeof(cl_uint), &offC);
+ CL_CHECK(err);
+ }
+
+ err = clEnqueueNDRangeKernel(commandQueues[0], *Kernels[0], 2, NULL, gs, wgsize, numEventsInWaitList, eventWaitList, NULL);
+
+ gs[0] = 16;
+ err |= clEnqueueNDRangeKernel(commandQueues[0], *Kernels[1], 2, NULL, gs, wgsize, 0, NULL, NULL);
+
+ gs[1] = 16;
+ gs[0] = GlobalX;
+ err |= clEnqueueNDRangeKernel(commandQueues[0], *Kernels[2], 2, NULL, gs, wgsize, 0, NULL, NULL);
+
+ gs[0] = 16; gs[1] = 16;
+ err |= clEnqueueNDRangeKernel(commandQueues[0], *Kernels[3], 2, NULL, gs, wgsize, 0, NULL, events);
+
+ if (err == 0)
+ return clblasSuccess;
+
+ }
+ }
+
+ return clblasNotImplemented;
+}
+
+clblasStatus GEMM_BRANCH_32(
+ clblasTranspose transA,
+ clblasTranspose transB,
+ cl_uint M, cl_uint N, cl_uint K,
+ float alpha,
+ cl_mem A, cl_uint offA, cl_uint lda,
+ cl_mem B, cl_uint offB, cl_uint ldb,
+ float beta,
+ cl_mem C, cl_uint offC, cl_uint ldc,
+ cl_uint numCommandQueues,
+ cl_command_queue *commandQueues,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *events,
+ bool &specialCaseHandled)
+{
+ const char *tileKernelSource = NULL;
+ cl_kernel *tileClKernel = NULL;
+ size_t tileKernelBinarySize = 0;
+ cl_int err;
+
+
+ const unsigned char *tileKernelBinary = NULL;
+
+ clblasStatus status;
+
+ if ((M * N < 1080 * 1080) && (M % 32 != 0 || N % 32 != 0) && (K%16==0))
+ {
+ // ((Mvalue - 1) / 32 + 1) * 16
+ size_t GlobalX = ((M - 1) / 32 + 1) * 16;
+ size_t GlobalY = ((N - 1) / 32 + 1) * 16;
+ size_t gs[2] = { GlobalX, GlobalY };
+ size_t wgsize[2] = { 16, 16 };
+
+ if (transA == clblasNoTrans && transB == clblasNoTrans)
+ {
+ specialCaseHandled = true;
+ tileKernelSource = sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_src;
+ tileClKernel = &sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_clKernel;
+ tileKernelBinary = sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_bin;
+ tileKernelBinarySize = sgemm_Col_NN_B1_MX032_NX032_KX16_BRANCH_binSize;
+
+ makeGemmKernel(tileClKernel, commandQueues[0], tileKernelSource, User_srcBuildOptions, &tileKernelBinary, &tileKernelBinarySize, User_binBuildOptions);
+
+ err = clSetKernelArg(*tileClKernel, 0, sizeof(cl_mem), &A);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 1, sizeof(cl_mem), &B);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 2, sizeof(cl_mem), &C);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 3, sizeof(cl_float), &alpha);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 4, sizeof(cl_float), &beta);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 5, sizeof(cl_uint), &M);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 6, sizeof(cl_uint), &N);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 7, sizeof(cl_uint), &K);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 8, sizeof(cl_uint), &lda);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 9, sizeof(cl_uint), &ldb);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 10, sizeof(cl_uint), &ldc);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 11, sizeof(cl_uint), &offA);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 12, sizeof(cl_uint), &offB);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 13, sizeof(cl_uint), &offC);
+ CL_CHECK(err);
+
+ err = clEnqueueNDRangeKernel(commandQueues[0], *tileClKernel, 2, NULL,
+ gs, wgsize, numEventsInWaitList, eventWaitList, &events[0]);
+
+ if (err == 0)
+ return clblasSuccess;
+ }
+ if (transA == clblasNoTrans && transB == clblasTrans)
+ {
+ specialCaseHandled = true;
+ tileKernelSource = sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_src;
+ tileClKernel = &sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_clKernel;
+ tileKernelBinary = sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_bin;
+ tileKernelBinarySize = sgemm_Col_NT_B1_MX032_NX032_KX16_BRANCH_binSize;
+
+ makeGemmKernel(tileClKernel, commandQueues[0], tileKernelSource, User_srcBuildOptions, &tileKernelBinary, &tileKernelBinarySize, User_binBuildOptions);
+
+ err = clSetKernelArg(*tileClKernel, 0, sizeof(cl_mem), &A);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 1, sizeof(cl_mem), &B);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 2, sizeof(cl_mem), &C);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 3, sizeof(cl_float), &alpha);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 4, sizeof(cl_float), &beta);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 5, sizeof(cl_uint), &M);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 6, sizeof(cl_uint), &N);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 7, sizeof(cl_uint), &K);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 8, sizeof(cl_uint), &lda);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 9, sizeof(cl_uint), &ldb);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 10, sizeof(cl_uint), &ldc);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 11, sizeof(cl_uint), &offA);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 12, sizeof(cl_uint), &offB);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 13, sizeof(cl_uint), &offC);
+ CL_CHECK(err);
+
+ err = clEnqueueNDRangeKernel(commandQueues[0], *tileClKernel, 2, NULL,
+ gs, wgsize, numEventsInWaitList, eventWaitList, &events[0]);
+
+ if (err == 0)
+ return clblasSuccess;
+ }
+ if (transA == clblasTrans && transB == clblasNoTrans)
+ {
+ specialCaseHandled = true;
+ tileKernelSource = sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_src;
+ tileClKernel = &sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_clKernel;
+ tileKernelBinary = sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_bin;
+ tileKernelBinarySize = sgemm_Col_TN_B1_MX032_NX032_KX16_BRANCH_binSize;
+
+ makeGemmKernel(tileClKernel, commandQueues[0], tileKernelSource, User_srcBuildOptions, &tileKernelBinary, &tileKernelBinarySize, User_binBuildOptions);
+
+ err = clSetKernelArg(*tileClKernel, 0, sizeof(cl_mem), &A);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 1, sizeof(cl_mem), &B);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 2, sizeof(cl_mem), &C);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 3, sizeof(cl_float), &alpha);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 4, sizeof(cl_float), &beta);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 5, sizeof(cl_uint), &M);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 6, sizeof(cl_uint), &N);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 7, sizeof(cl_uint), &K);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 8, sizeof(cl_uint), &lda);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 9, sizeof(cl_uint), &ldb);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 10, sizeof(cl_uint), &ldc);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 11, sizeof(cl_uint), &offA);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 12, sizeof(cl_uint), &offB);
+ CL_CHECK(err);
+ err = clSetKernelArg(*tileClKernel, 13, sizeof(cl_uint), &offC);
+ CL_CHECK(err);
+
+ err = clEnqueueNDRangeKernel(commandQueues[0], *tileClKernel, 2, NULL,
+ gs, wgsize, numEventsInWaitList, eventWaitList, &events[0]);
+
+ if (err == 0)
+ return clblasSuccess;
+ }
+ }
+
+ return clblasNotImplemented;
+}
+
+template<>
+clblasStatus
+GemmSpecialCases<float>(clblasOrder order,
+clblasTranspose transA,
+clblasTranspose transB,
+cl_uint M, cl_uint N, cl_uint K,
+float alpha,
+cl_mem A, cl_uint offA, cl_uint lda,
+cl_mem B, cl_uint offB, cl_uint ldb,
+float beta,
+cl_mem C, cl_uint offC, cl_uint ldc,
+cl_uint numCommandQueues,
+cl_command_queue *commandQueues,
+cl_uint numEventsInWaitList,
+const cl_event *eventWaitList,
+cl_event *events,
+bool &specialCaseHandled)
+{
+
+ if (order == clblasRowMajor)
+ return clblasNotImplemented;
+
+ clblasStatus status;
+
+ //handles big multiples of 1024
+ status = GEMM_mod1024(transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events,
+ specialCaseHandled);
+ if (specialCaseHandled)
+ return status;
+
+ //handles mod32 but not mod64
+ status = GEMM_SPLIT64_32(transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events,
+ specialCaseHandled);
+ if (specialCaseHandled)
+ return status;
+
+ //handles middle range sgemm (M*N<1080*1080) that are not mod32 (M%32!=0 || N%32!=0)
+ //use 32x32 micro tile kernels with branch statement within kernels
+ status = GEMM_BRANCH_32(transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events,
+ specialCaseHandled);
+ if (specialCaseHandled)
+ return status;
+
+
+ return clblasNotImplemented;
+}
+
+template<>
+clblasStatus
+GemmSpecialCases<double>(clblasOrder order,
+clblasTranspose transA,
+clblasTranspose transB,
+cl_uint M, cl_uint N, cl_uint K,
+double alpha,
+cl_mem A, cl_uint offA, cl_uint lda,
+cl_mem B, cl_uint offB, cl_uint ldb,
+double beta,
+cl_mem C, cl_uint offC, cl_uint ldc,
+cl_uint numCommandQueues,
+cl_command_queue *commandQueues,
+cl_uint numEventsInWaitList,
+const cl_event *eventWaitList,
+cl_event *events,
+bool &specialCaseHandled)
+{
+ return clblasNotImplemented;
+}
+
+template<>
+clblasStatus
+GemmSpecialCases<FloatComplex>(clblasOrder order,
+clblasTranspose transA,
+clblasTranspose transB,
+cl_uint M, cl_uint N, cl_uint K,
+FloatComplex alpha,
+cl_mem A, cl_uint offA, cl_uint lda,
+cl_mem B, cl_uint offB, cl_uint ldb,
+FloatComplex beta,
+cl_mem C, cl_uint offC, cl_uint ldc,
+cl_uint numCommandQueues,
+cl_command_queue *commandQueues,
+cl_uint numEventsInWaitList,
+const cl_event *eventWaitList,
+cl_event *events,
+bool &specialCaseHandled)
+{
+ return clblasNotImplemented;
+}
+
+template<>
+clblasStatus
+GemmSpecialCases<DoubleComplex>(clblasOrder order,
+clblasTranspose transA,
+clblasTranspose transB,
+cl_uint M, cl_uint N, cl_uint K,
+DoubleComplex alpha,
+cl_mem A, cl_uint offA, cl_uint lda,
+cl_mem B, cl_uint offB, cl_uint ldb,
+DoubleComplex beta,
+cl_mem C, cl_uint offC, cl_uint ldc,
+cl_uint numCommandQueues,
+cl_command_queue *commandQueues,
+cl_uint numEventsInWaitList,
+const cl_event *eventWaitList,
+cl_event *events,
+bool &specialCaseHandled)
+{
+ return clblasNotImplemented;
+}
\ No newline at end of file
diff --git a/src/library/blas/specialCases/include/GemmSpecialCases.h b/src/library/blas/specialCases/include/GemmSpecialCases.h
new file mode 100644
index 0000000..c669c5a
--- /dev/null
+++ b/src/library/blas/specialCases/include/GemmSpecialCases.h
@@ -0,0 +1,42 @@
+/* ************************************************************************
+* Copyright 2015 Advanced Micro Devices, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+* ************************************************************************/
+
+#ifndef CLBLAS_GEMM_SPECIAL_CASES_H
+#define CLBLAS_GEMM_SPECIAL_CASES_H
+
+#include <clBLAS.h>
+#include <stdio.h>
+#include <assert.h>
+
+template<typename Precision>
+clblasStatus
+GemmSpecialCases(clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ cl_uint M, cl_uint N, cl_uint K,
+ Precision alpha,
+ cl_mem A, cl_uint offA, cl_uint lda,
+ cl_mem B, cl_uint offB, cl_uint ldb,
+ Precision beta,
+ cl_mem C, cl_uint offC, cl_uint ldc,
+ cl_uint numCommandQueues,
+ cl_command_queue *commandQueues,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *events,
+ bool &specialCaseHandled);
+
+#endif
\ No newline at end of file
diff --git a/src/library/blas/xgemm.cc b/src/library/blas/xgemm.cc
index be5290c..1c66ceb 100644
--- a/src/library/blas/xgemm.cc
+++ b/src/library/blas/xgemm.cc
@@ -14,140 +14,587 @@
* limitations under the License.
* ************************************************************************/
-
+#include <stdio.h>
#include <string.h>
#include <clBLAS.h>
+#include "AutoGemmIncludes/AutoGemmKernelSelection.h"
+#include "GemmSpecialCases.h"
+
+ #include <functor.h>
+// #include <functor_selector.h>
+#include "xgemm.h"
+
+/******************************************************************************
+ * Row major -> column major
+ *****************************************************************************/
+static void force_gemm_column_major(
+ clblasOrder &order,
+ clblasTranspose &transA,
+ clblasTranspose &transB,
+ cl_uint &M,
+ cl_uint &N,
+ cl_uint &offA,
+ cl_uint &offB,
+ cl_uint &lda,
+ cl_uint &ldb,
+ cl_mem &A,
+ cl_mem &B )
+{
+ if (order == clblasRowMajor) {
+ std::swap(transA , transB);
+ std::swap(M , N);
+ std::swap(offA , offB);
+ std::swap(lda , ldb);
+ std::swap(A , B);
+ order = clblasColumnMajor;
+ }
+}
-#include <functor.h>
-#include <functor_selector.h>
-
-// Transform a gemm in clblasRowMajor into a gemm in clblasColumnMajor:
-//
-// The idea is basically that
-// C = A*B + C
-// can be computed as
-// C' = (A*B + C)'
-// = B'*A' + C'
-// And since changing the order is basically a transpose on each matrix,
-// the formula becomes with the new order
-// C = B*A + C
-//
-// When enabled, only the ColumnMajor kernels need to be implemented
-// for all GEMM
-//
-
-#define FORCE_COLUMN_MAJOR 1
-
-#if FORCE_COLUMN_MAJOR
-template <typename Args>
-static void force_gemm_column_major(Args & args)
+/******************************************************************************
+ * Check OpenCL Errors
+ *****************************************************************************/
+#define CL_CHECK(RET) \
+ if(RET != CL_SUCCESS) { \
+ printf("OpenCL error %i on line %u\n", RET, __LINE__); \
+ assert(false); \
+ }
+
+const static unsigned int numGemmKernelArgs = 14;
+void *gemmKernelArgs[numGemmKernelArgs];
+size_t gemmKernelArgSizes[numGemmKernelArgs];
+
+
+/******************************************************************************
+ * Is beta zero for optimization
+ *****************************************************************************/
+template <typename Precision>
+bool isZero(Precision value);
+template<>
+bool isZero<float>( float value ) {
+ return value == 0;
+};
+template<>
+bool isZero<double>( double value ) {
+ return value == 0;
+};
+template<>
+bool isZero<FloatComplex>( FloatComplex value ) {
+ return CREAL(value) == 0 && CIMAG(value) == 0;
+};
+template<>
+bool isZero<DoubleComplex>( DoubleComplex value ) {
+ return CREAL(value) == 0 && CIMAG(value) == 0;
+};
+
+
+
+/******************************************************************************
+ * Make Gemm Kernel
+ *****************************************************************************/
+void makeGemmKernel(
+ cl_kernel *clKernel,
+ cl_command_queue clQueue,
+ const char *kernelSource,
+ const char *sourceBuildOptions,
+ const unsigned char **kernelBinary,
+ size_t *kernelBinarySize,
+ const char *binaryBuildOptions)
{
- if (args.order == clblasRowMajor)
- {
- std::swap(args.transA , args.transB);
- std::swap(args.M , args.N);
- std::swap(args.offA , args.offB);
- std::swap(args.lda , args.ldb);
- std::swap(args.A , args.B);
- args.order = clblasColumnMajor;
+ cl_int err;
+ if (*clKernel) {
+ // kernel has already been built, return
+#ifdef AUTOGEMM_PRINT_DEBUG
+ // get kernel name
+ size_t kernelNameLength;
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ sizeof(kernelNameLength),
+ NULL,
+ &kernelNameLength );
+ CL_CHECK(err)
+ char *kernelName = new char[kernelNameLength];
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ kernelNameLength*sizeof(char),
+ kernelName,
+ NULL );
+ CL_CHECK(err)
+ printf("makeGemmKernel: \"%s\" already built; returning.\n", kernelName);
+ delete[] kernelName;
+#endif
+ return;
+ } else {
+ // kernel has not been built, so build it (from binary, preferably)
+ cl_context clContext;
+ cl_device_id clDevice;
+ err = clGetCommandQueueInfo( clQueue, CL_QUEUE_CONTEXT, sizeof(clContext), &clContext, NULL);
+ CL_CHECK(err)
+ err = clGetCommandQueueInfo( clQueue, CL_QUEUE_DEVICE, sizeof(clDevice), &clDevice, NULL);
+ CL_CHECK(err)
+ cl_program clProgram;
+ cl_int clBinaryStatus;
+ if (*kernelBinary) {
+#ifdef AUTOGEMM_PRINT_DEBUG
+ printf("makeGemmKernel: pre-compiled binary found: %llu bytes\n", *kernelBinarySize);
+#endif
+ clProgram = clCreateProgramWithBinary(
+ clContext,
+ 1, &clDevice,
+ kernelBinarySize, kernelBinary,
+ &clBinaryStatus, &err );
+ CL_CHECK(err)
+ err = clBuildProgram(
+ clProgram,
+ 1, &clDevice,
+ binaryBuildOptions, NULL, NULL );
+ CL_CHECK(err)
+ } else {
+ clProgram = clCreateProgramWithSource(
+ clContext,
+ 1, &kernelSource,
+ NULL, &err );
+ CL_CHECK(err)
+ err = clBuildProgram(
+ clProgram,
+ 1, &clDevice,
+ sourceBuildOptions, NULL, NULL );
+ CL_CHECK(err)
}
-}
+ err = clCreateKernelsInProgram(
+ clProgram,
+ 1, clKernel,
+ NULL );
+ CL_CHECK(err)
+
+#ifdef AUTOGEMM_PRINT_DEBUG
+ // get kernel name
+ size_t kernelNameLength;
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ sizeof(kernelNameLength),
+ NULL,
+ &kernelNameLength );
+ CL_CHECK(err)
+ char *kernelName = new char[kernelNameLength];
+ err = clGetKernelInfo(
+ *clKernel,
+ CL_KERNEL_FUNCTION_NAME,
+ kernelNameLength*sizeof(char),
+ kernelName,
+ NULL );
+ CL_CHECK(err)
+ printf("makeGemmKernel: \"%s\" now built; returning.\n", kernelName);
+ delete[] kernelName;
#endif
+ }
+}
-//
-// This file provide the public clBLAS API for
-//
-// clblasSgemm()
-// clblasDgemm()
-// clblasCgemm()
-// clblasZgemm()
-//
-// using functors
-//
-// Potential optimizations:
-//
-// - Check the values of alpha, beta, M, N and K to
-// transform the gemm into an equivalent but cheaper
-// scal or gemv where possible.
-//
-// - Get rid of the 'order' argument assuming that
-// row-major is equivalent to the transpose of column-major.
-// That is
-//
-// C = alpha * A * B + beta * C
-//
-// is equivalent to
-//
-// C' = alpha * B' * A' + beta * C'
-//
-// and, when considering the opposite order, is equivalent to
-//
-// C = alpha * B * A + beta * C
-//
-// By applying that transformation early, the functors implementing
-// the GEMMs only have to consider one of the two cases.
-//
-
-
-extern "C"
+
+/******************************************************************************
+ * Enqueue Gemm Kernel
+ *****************************************************************************/
+ void enqueueGemmKernel(
+ cl_command_queue clQueue,
+ cl_kernel clKernel,
+ void **kernelArgs,
+ size_t *kernelArgSizes,
+ unsigned int numKernelArgs,
+ const size_t *globalWorkSize,
+ const size_t *localWorkSize,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *clEvent)
+ {
+ for (unsigned int i = 0; i < numKernelArgs; i++) {
+ CL_CHECK( clSetKernelArg( clKernel, i, kernelArgSizes[i], kernelArgs[i]) )
+ }
+ /*printf("global={%llu, %llu} local={%llu, %llu}\n",
+ globalWorkSize[0], globalWorkSize[1],
+ localWorkSize[0], localWorkSize[1] );*/
+ CL_CHECK( clEnqueueNDRangeKernel( clQueue, clKernel,
+ 2, NULL, globalWorkSize, localWorkSize,
+ numEventsInWaitList, eventWaitList, clEvent ) )
+ }
+
+
+/******************************************************************************
+ * get precision string
+ *****************************************************************************/
+template<typename Precision>
+char * getPrecision();
+template<> char * getPrecision<float>() { return "s"; }
+template<> char * getPrecision<double>() { return "d"; }
+template<> char * getPrecision<FloatComplex>() { return "c"; }
+template<> char * getPrecision<DoubleComplex>() { return "z"; }
+
+
+/******************************************************************************
+ * convert ConjTrans -> Trans for real
+ *****************************************************************************/
+template<typename Precision>
+clblasTranspose correctTranspose(clblasTranspose trans);
+template<> clblasTranspose correctTranspose<float>( clblasTranspose trans) { return (trans==clblasConjTrans) ? clblasTrans : trans; }
+template<> clblasTranspose correctTranspose<double>( clblasTranspose trans) { return (trans==clblasConjTrans) ? clblasTrans : trans; }
+template<> clblasTranspose correctTranspose<FloatComplex>( clblasTranspose trans) { return trans; }
+template<> clblasTranspose correctTranspose<DoubleComplex>( clblasTranspose trans) { return trans; }
+
+
+/******************************************************************************
+ * templated Gemm
+ *****************************************************************************/
+template<typename Precision>
clblasStatus
-clblasSgemm( clblasOrder order,
- clblasTranspose transA,
- clblasTranspose transB,
- size_t M, size_t N, size_t K,
- cl_float alpha,
- const cl_mem A, size_t offA, size_t lda,
- const cl_mem B, size_t offB, size_t ldb,
- cl_float beta,
- cl_mem C, size_t offC, size_t ldc,
- cl_uint numCommandQueues,
- cl_command_queue *commandQueues,
- cl_uint numEventsInWaitList,
- const cl_event *eventWaitList,
- cl_event *events)
+clblasGemm(
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ size_t iM, size_t iN, size_t iK,
+ Precision alpha,
+ const cl_mem iA, size_t iOffA, size_t iLda,
+ const cl_mem iB, size_t iOffB, size_t iLdb,
+ Precision beta,
+ cl_mem C, size_t iOffC, size_t iLdc,
+ cl_uint numCommandQueues,
+ cl_command_queue *commandQueues,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *events)
{
- CHECK_QUEUES(numCommandQueues, commandQueues);
- CHECK_EVENTS(numEventsInWaitList, eventWaitList);
- CHECK_MATRIX_A(TYPE_FLOAT, order, transA, A, M, K, offA, lda);
- CHECK_MATRIX_B(TYPE_FLOAT, order, transB, B, K, N, offB, ldb);
- CHECK_MATRIX_C(TYPE_FLOAT, order, clblasNoTrans, C, M, N, offC, ldc);
-
- if ( numCommandQueues>1 )
- {
- numCommandQueues = 1 ; // No support for multi-device (yet)
- }
-
- cl_command_queue queue = commandQueues[0];
-
- clblasSgemmFunctor::Args args(order,
- transA,
- transB,
- M, N, K,
- alpha,
- A, offA, lda,
- B, offB, ldb,
- beta,
- C, offC, ldc,
- queue,
- numEventsInWaitList,
- eventWaitList,
- events);
-
-#if FORCE_COLUMN_MAJOR
- force_gemm_column_major(args);
+ // cast types to opencl types
+ cl_mem A = iA;
+ cl_mem B = iB;
+ cl_uint M = static_cast<cl_uint>( iM );
+ cl_uint N = static_cast<cl_uint>( iN );
+ cl_uint K = static_cast<cl_uint>( iK );
+ cl_uint offA = static_cast<cl_uint>( iOffA );
+ cl_uint offB = static_cast<cl_uint>( iOffB );
+ cl_uint offC = static_cast<cl_uint>( iOffC );
+ cl_uint lda = static_cast<cl_uint>( iLda );
+ cl_uint ldb = static_cast<cl_uint>( iLdb );
+ cl_uint ldc = static_cast<cl_uint>( iLdc );
+
+ transA = correctTranspose<Precision>(transA);
+ transB = correctTranspose<Precision>(transB);
+ // if debug build, validate input
+ // CHECK_QUEUES(numCommandQueues, commandQueues);
+ // CHECK_EVENTS(numEventsInWaitList, eventWaitList);
+ // CHECK_MATRIX_A(Precision, order, transA, A, M, K, offA, lda);
+ // CHECK_MATRIX_B(Precision, order, transB, B, K, N, offB, ldb);
+ // CHECK_MATRIX_C(Precision, order, clblasNoTrans, C, M, N, offC, ldc);
+ force_gemm_column_major( order, transA, transB,
+ M, N, offA, offB, lda, ldb, A, B );
+
+
+
+/******************************************************************************
+ * Handle Special Cases
+ *
+ * 1) sgemm NT where lda, ldb are big multiples of 1024 starting from 4096
+ *
+ * 2) sgemm NT where M and N are within middle range
+ * and are mod32 but not mod96 or mod64
+ *
+ *****************************************************************************/
+
+ bool specialCaseHandled = false;
+
+ clblasStatus SpecialCaseStatus = GemmSpecialCases<Precision>(order,
+ transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events,
+ specialCaseHandled);
+
+ if (specialCaseHandled)
+ return SpecialCaseStatus;
+
+
+/******************************************************************************
+ * Optimal num elements per thread
+ *****************************************************************************/
+ cl_int err;
+ cl_device_id clDevice;
+ err = clGetCommandQueueInfo( commandQueues[0], CL_QUEUE_DEVICE, sizeof(clDevice), &clDevice, NULL);
+ CL_CHECK(err)
+ cl_uint clDeviceNumCUs;
+ err = clGetDeviceInfo( clDevice, CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(clDeviceNumCUs), &clDeviceNumCUs, NULL);
+ CL_CHECK(err)
+ unsigned int deviceIdealNumThreads = (8 /*waves per CU*/)*(64 /*threads per wave*/)*clDeviceNumCUs;
+ float optimalNumElementsPerThread = ((float)M*N) / deviceIdealNumThreads;
+ //optimalNumElementsPerThread = 32;
+ bool betaNonZero = !isZero(beta);
+
+#ifdef AUTOGEMM_PRINT_DEBUG
+ printf("%sgemm_%3s_%s%s_B%u_%llux%llux%llu\n",
+ getPrecision<Precision>(),
+ order==clblasColumnMajor ? "Col" : "Row",
+ transA==clblasNoTrans ? "N" : transA==clblasTrans ? "T" : "C",
+ transB==clblasNoTrans ? "N" : transB==clblasTrans ? "T" : "C",
+ betaNonZero ? 1 : 0,
+ iM, iN, iK );
#endif
- clblasFunctorSelector * fselector = clblasFunctorSelector::find(queue);
-
- clblasSgemmFunctor * functor = fselector->select_sgemm_specific(args);
+/******************************************************************************
+ * Select kernel
+ *****************************************************************************/
+ const char *tileKernelSource = NULL;
+ const char *rowKernelSource = NULL;
+ const char *colKernelSource = NULL;
+ const char *cornerKernelSource = NULL;
+ const char *sourceBuildOptions = NULL;
+ const unsigned char *tileKernelBinary = NULL;
+ const unsigned char *rowKernelBinary = NULL;
+ const unsigned char *colKernelBinary = NULL;
+ const unsigned char *cornerKernelBinary = NULL;
+ size_t *tileKernelBinarySize = 0;
+ size_t *rowKernelBinarySize = 0;
+ size_t *colKernelBinarySize = 0;
+ size_t *cornerKernelBinarySize = 0;
+ const char *binaryBuildOptions = NULL;
+ cl_kernel *tileClKernel = NULL;
+ cl_kernel *rowClKernel = NULL;
+ cl_kernel *colClKernel = NULL;
+ cl_kernel *cornerClKernel = NULL;
+ unsigned int workGroupNumRows;
+ unsigned int workGroupNumCols;
+ unsigned int microTileNumRows;
+ unsigned int microTileNumCols;
+ unsigned int unroll;
+ gemmSelectKernel<Precision>(
+ order, transA, transB,
+ iM, iN, iK,
+ betaNonZero,
+ optimalNumElementsPerThread,
+ &tileKernelSource,
+ &rowKernelSource,
+ &colKernelSource,
+ &cornerKernelSource,
+ &sourceBuildOptions,
+ &tileKernelBinary,
+ &rowKernelBinary,
+ &colKernelBinary,
+ &cornerKernelBinary,
+ &tileKernelBinarySize,
+ &rowKernelBinarySize,
+ &colKernelBinarySize,
+ &cornerKernelBinarySize,
+ &binaryBuildOptions,
+ &tileClKernel,
+ &rowClKernel,
+ &colClKernel,
+ &cornerClKernel,
+ &workGroupNumRows,
+ &workGroupNumCols,
+ µTileNumRows,
+ µTileNumCols,
+ &unroll);
+ // make sure gemmSelectKernel found a valid kernel
+ if (!tileKernelSource) {
+ printf("ERROR: gemmSelectKernel() couldn't find kernel(s) for { order=%s, transA=%s, transB=%s, M=%llu, N=%llu, K=%llu, beta=%u, onept=%f }\n",
+ order==clblasColumnMajor ? "ColMajor" : "RowMajor",
+ transA==clblasNoTrans ? "N" : transA==clblasTrans ? "T" : "C",
+ transB==clblasNoTrans ? "N" : transB==clblasTrans ? "T" : "C",
+ M, N, K,
+ betaNonZero ? 1 : 0,
+ optimalNumElementsPerThread );
+ gemmSelectKernel<Precision>(
+ order,
+ transA,
+ transB,
+ M,
+ N,
+ K,
+ betaNonZero,
+ optimalNumElementsPerThread,
+ &tileKernelSource,
+ &rowKernelSource,
+ &colKernelSource,
+ &cornerKernelSource,
+ &sourceBuildOptions,
+ &tileKernelBinary,
+ &rowKernelBinary,
+ &colKernelBinary,
+ &cornerKernelBinary,
+ &tileKernelBinarySize,
+ &rowKernelBinarySize,
+ &colKernelBinarySize,
+ &cornerKernelBinarySize,
+ &binaryBuildOptions,
+ &tileClKernel,
+ &rowClKernel,
+ &colClKernel,
+ &cornerClKernel,
+ &workGroupNumRows,
+ &workGroupNumCols,
+ µTileNumRows,
+ µTileNumCols,
+ &unroll);
+ return clblasNotImplemented;
+ }
+
+
+ unsigned int macroTileNumRows = workGroupNumRows*microTileNumRows;
+ unsigned int macroTileNumCols = workGroupNumCols*microTileNumCols;
+ bool needTileKernel = M/macroTileNumRows > 0
+ && N/macroTileNumCols > 0;
+ bool needRowKernel = M%macroTileNumRows > 0 && N/macroTileNumCols > 0;
+ bool needColKernel = N%macroTileNumCols > 0 && M/macroTileNumRows > 0;
+ bool needCornerKernel = M%macroTileNumRows > 0 && N%macroTileNumCols > 0;
+#if 0
+ printf("For M,N,K = %u,%u,%u and %u CUs selected tile is wg=%ux%u, microTile=%ux%u, macroTile=%ux%u kernelsNeeded=%u,%u,%u,%u\n",
+ M, N, K, clDeviceNumCUs,
+ workGroupNumRows, workGroupNumCols,
+ microTileNumRows, microTileNumCols,
+ macroTileNumRows, macroTileNumCols,
+ needTileKernel ? 1 : 0,
+ needRowKernel ? 1 : 0,
+ needColKernel ? 1 : 0,
+ needCornerKernel ? 1 : 0
+ );
+#endif
- clblasStatus res = functor->execute(args);
+/******************************************************************************
+ * Build kernels
+ *****************************************************************************/
+ if (needTileKernel) makeGemmKernel( tileClKernel, commandQueues[0], tileKernelSource, sourceBuildOptions, &tileKernelBinary, tileKernelBinarySize, binaryBuildOptions);
+ if (needRowKernel) makeGemmKernel( rowClKernel, commandQueues[0], rowKernelSource, sourceBuildOptions, &rowKernelBinary, rowKernelBinarySize, binaryBuildOptions);
+ if (needColKernel) makeGemmKernel( colClKernel, commandQueues[0], colKernelSource, sourceBuildOptions, &colKernelBinary, colKernelBinarySize, binaryBuildOptions);
+ if (needCornerKernel) makeGemmKernel(cornerClKernel, commandQueues[0], cornerKernelSource, sourceBuildOptions, &cornerKernelBinary, cornerKernelBinarySize, binaryBuildOptions);
+ const size_t localWorkSize[2] = { workGroupNumRows, workGroupNumCols };
+ unsigned int numKernelsEnqueued = 0;
+
+/******************************************************************************
+ * Gather kernel arguments
+ *****************************************************************************/
+ gemmKernelArgs[ 0] = &A; gemmKernelArgSizes[ 0] = sizeof(cl_mem);
+ gemmKernelArgs[ 1] = &B; gemmKernelArgSizes[ 1] = sizeof(cl_mem);
+ gemmKernelArgs[ 2] = &C; gemmKernelArgSizes[ 2] = sizeof(cl_mem);
+ gemmKernelArgs[ 3] = α gemmKernelArgSizes[ 3] = sizeof(Precision);
+ gemmKernelArgs[ 4] = β gemmKernelArgSizes[ 4] = sizeof(Precision);
+ gemmKernelArgs[ 5] = &M; gemmKernelArgSizes[ 5] = sizeof(cl_uint);
+ gemmKernelArgs[ 6] = &N; gemmKernelArgSizes[ 6] = sizeof(cl_uint);
+ gemmKernelArgs[ 7] = &K; gemmKernelArgSizes[ 7] = sizeof(cl_uint);
+ gemmKernelArgs[ 8] = &lda; gemmKernelArgSizes[ 8] = sizeof(cl_uint);
+ gemmKernelArgs[ 9] = &ldb; gemmKernelArgSizes[ 9] = sizeof(cl_uint);
+ gemmKernelArgs[10] = &ldc; gemmKernelArgSizes[10] = sizeof(cl_uint);
+ gemmKernelArgs[11] = &offA; gemmKernelArgSizes[11] = sizeof(cl_uint);
+ gemmKernelArgs[12] = &offB; gemmKernelArgSizes[12] = sizeof(cl_uint);
+ gemmKernelArgs[13] = &offC; gemmKernelArgSizes[13] = sizeof(cl_uint);
+
+
+/******************************************************************************
+ * Enqueue Tile kernel
+ *****************************************************************************/
+ if (needTileKernel) {
+ //printf("enqueueing tile kernel\n");
+ size_t globalWorkSize[2] = {(M/macroTileNumRows)*workGroupNumRows, (N/macroTileNumCols)*workGroupNumCols };
+ enqueueGemmKernel( commandQueues[numKernelsEnqueued%numCommandQueues], *tileClKernel,
+ gemmKernelArgs, gemmKernelArgSizes, numGemmKernelArgs,
+ globalWorkSize, localWorkSize,
+ numEventsInWaitList, eventWaitList,
+ &events[numKernelsEnqueued%numCommandQueues] );
+ numKernelsEnqueued++;
+ }
+
+/******************************************************************************
+ * Enqueue Row kernel
+ *****************************************************************************/
+ if (needRowKernel) {
+ //printf("enqueueing row kernel\n");
+ size_t globalWorkSize[2] = {1*workGroupNumRows, (N/macroTileNumCols)*workGroupNumCols };
+ enqueueGemmKernel( commandQueues[numKernelsEnqueued%numCommandQueues], *rowClKernel,
+ gemmKernelArgs, gemmKernelArgSizes, numGemmKernelArgs,
+ globalWorkSize, localWorkSize,
+ numEventsInWaitList, eventWaitList,
+ &events[numKernelsEnqueued%numCommandQueues] );
+ numKernelsEnqueued++;
+ }
+
+/******************************************************************************
+ * Enqueue Col kernel
+ *****************************************************************************/
+ if (needColKernel) {
+ //printf("enqueueing col kernel\n");
+ size_t globalWorkSize[2] = { (M/macroTileNumRows)*workGroupNumRows, 1*workGroupNumCols };
+ enqueueGemmKernel( commandQueues[numKernelsEnqueued%numCommandQueues], *colClKernel,
+ gemmKernelArgs, gemmKernelArgSizes, numGemmKernelArgs,
+ globalWorkSize, localWorkSize,
+ numEventsInWaitList, eventWaitList,
+ &events[numKernelsEnqueued%numCommandQueues] );
+ numKernelsEnqueued++;
+ }
+
+/******************************************************************************
+ * Enqueue Corner kernel
+ *****************************************************************************/
+ if (needCornerKernel) {
+ //printf("enqueueing corner kernel\n");
+ size_t globalWorkSize[2] = { 1*workGroupNumRows, 1*workGroupNumCols };
+ enqueueGemmKernel( commandQueues[numKernelsEnqueued%numCommandQueues], *cornerClKernel,
+ gemmKernelArgs, gemmKernelArgSizes, numGemmKernelArgs,
+ globalWorkSize, localWorkSize,
+ numEventsInWaitList, eventWaitList,
+ &events[numKernelsEnqueued%numCommandQueues] );
+ numKernelsEnqueued++;
+ }
+
+ return clblasSuccess;
+}
- functor->release();
- return res;
+/******************************************************************************
+ * SGEMM API call
+ *****************************************************************************/
+extern "C"
+clblasStatus
+clblasSgemm(
+ clblasOrder order,
+ clblasTranspose transA,
+ clblasTranspose transB,
+ size_t M, size_t N, size_t K,
+ cl_float alpha,
+ const cl_mem A, size_t offA, size_t lda,
+ const cl_mem B, size_t offB, size_t ldb,
+ cl_float beta,
+ cl_mem C, size_t offC, size_t ldc,
+ cl_uint numCommandQueues,
+ cl_command_queue *commandQueues,
+ cl_uint numEventsInWaitList,
+ const cl_event *eventWaitList,
+ cl_event *events)
+{
+ return clblasGemm(
+ order,
+ transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events);
}
+/******************************************************************************
+ * DGEMM API call
+ *****************************************************************************/
extern "C"
clblasStatus
clblasDgemm( clblasOrder order,
@@ -165,49 +612,26 @@ clblasDgemm( clblasOrder order,
const cl_event *eventWaitList,
cl_event *events)
{
-//printf("dgemm M=%i,N=%i,K=%i,lda=%i,ldb=%i,ldc=%i\n", M, N, K, lda, ldb, ldc);
- CHECK_QUEUES(numCommandQueues, commandQueues);
- CHECK_EVENTS(numEventsInWaitList, eventWaitList);
- CHECK_MATRIX_A(TYPE_DOUBLE, order, transA, A, M, K, offA, lda);
- CHECK_MATRIX_B(TYPE_DOUBLE, order, transB, B, K, N, offB, ldb);
- CHECK_MATRIX_C(TYPE_DOUBLE, order, clblasNoTrans, C, M, N, offC, ldc);
-
- if ( numCommandQueues>1 )
- {
- numCommandQueues = 1 ; // No support for multi-device (yet)
- }
-
- cl_command_queue queue = commandQueues[0];
-
- clblasDgemmFunctor::Args args(order,
- transA,
- transB,
- M, N, K,
- alpha,
- A, offA, lda,
- B, offB, ldb,
- beta,
- C, offC, ldc,
- queue,
- numEventsInWaitList,
- eventWaitList,
- events);
-
-#if FORCE_COLUMN_MAJOR
- force_gemm_column_major(args);
-#endif
-
- clblasFunctorSelector * fselector = clblasFunctorSelector::find(queue);
-
- clblasDgemmFunctor * functor = fselector->select_dgemm_specific(args);
-
- clblasStatus res = functor->execute(args);
-
- functor->release();
-
- return res;
+ return clblasGemm(
+ order,
+ transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events);
}
+/******************************************************************************
+ * CGEMM API call
+ *****************************************************************************/
extern "C"
clblasStatus
clblasCgemm(
@@ -226,48 +650,26 @@ clblasCgemm(
const cl_event *eventWaitList,
cl_event *events)
{
- CHECK_QUEUES(numCommandQueues, commandQueues);
- CHECK_EVENTS(numEventsInWaitList, eventWaitList);
- CHECK_MATRIX_A(TYPE_COMPLEX_FLOAT, order, transA, A, M, K, offA, lda);
- CHECK_MATRIX_B(TYPE_COMPLEX_FLOAT, order, transB, B, K, N, offB, ldb);
- CHECK_MATRIX_C(TYPE_COMPLEX_FLOAT, order, clblasNoTrans, C, M, N, offC, ldc);
-
- if ( numCommandQueues>1 )
- {
- numCommandQueues = 1 ; // No support for multi-device (yet)
- }
-
- cl_command_queue queue = commandQueues[0];
-
- clblasCgemmFunctor::Args args(order,
- transA,
- transB,
- M, N, K,
- alpha,
- A, offA, lda,
- B, offB, ldb,
- beta,
- C, offC, ldc,
- queue,
- numEventsInWaitList,
- eventWaitList,
- events);
-
-#if FORCE_COLUMN_MAJOR
- force_gemm_column_major(args);
-#endif
-
- clblasFunctorSelector * fselector = clblasFunctorSelector::find(queue);
-
- clblasCgemmFunctor * functor = fselector->select_cgemm_specific(args);
-
- clblasStatus res = functor->execute(args);
-
- functor->release();
-
- return res;
+ return clblasGemm(
+ order,
+ transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events);
}
+/******************************************************************************
+ * ZGEMM API
+ *****************************************************************************/
extern "C"
clblasStatus
clblasZgemm(
@@ -286,44 +688,19 @@ clblasZgemm(
const cl_event *eventWaitList,
cl_event *events)
{
- CHECK_QUEUES(numCommandQueues, commandQueues);
- CHECK_EVENTS(numEventsInWaitList, eventWaitList);
- CHECK_MATRIX_A(TYPE_COMPLEX_DOUBLE, order, transA, A, M, K, offA, lda);
- CHECK_MATRIX_B(TYPE_COMPLEX_DOUBLE, order, transB, B, K, N, offB, ldb);
- CHECK_MATRIX_C(TYPE_COMPLEX_DOUBLE, order, clblasNoTrans, C, M, N, offC, ldc);
-
- if ( numCommandQueues>1 )
- {
- numCommandQueues = 1 ; // No support for multi-device (yet)
- }
-
- cl_command_queue queue = commandQueues[0];
-
- clblasZgemmFunctor::Args args(order,
- transA,
- transB,
- M, N, K,
- alpha,
- A, offA, lda,
- B, offB, ldb,
- beta,
- C, offC, ldc,
- queue,
- numEventsInWaitList,
- eventWaitList,
- events);
-
-#if FORCE_COLUMN_MAJOR
- force_gemm_column_major(args);
-#endif
-
- clblasFunctorSelector * fselector = clblasFunctorSelector::find(queue);
-
- clblasZgemmFunctor * functor = fselector->select_zgemm_specific(args);
-
- clblasStatus res = functor->execute(args);
-
- functor->release();
-
- return res;
+ return clblasGemm(
+ order,
+ transA,
+ transB,
+ M, N, K,
+ alpha,
+ A, offA, lda,
+ B, offB, ldb,
+ beta,
+ C, offC, ldc,
+ numCommandQueues,
+ commandQueues,
+ numEventsInWaitList,
+ eventWaitList,
+ events);
}
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index 9ecfd13..907cac0 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -385,7 +385,7 @@ if( GTEST_FOUND )
ARCHIVE DESTINATION lib${SUFFIX_LIB}/import
)
- get_target_property( testLocation test-correctness LOCATION )
+ #get_target_property( testLocation test-correctness LOCATION )
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/copyTestDependencies.cmake.in"
diff --git a/src/tests/common.cpp b/src/tests/common.cpp
index 759a588..d0f21ba 100644
--- a/src/tests/common.cpp
+++ b/src/tests/common.cpp
@@ -106,21 +106,28 @@ printTestParams(
size_t offC,
size_t ldc)
{
- ::std::cerr << orderStr(order) << ", " << transStr(transA) << ", " <<
- transStr(transB) << ::std::endl;
- ::std::cerr << "M = " << M << ", N = " << N << ", K = " << K << ::std::endl;
- ::std::cerr << "offA = " << offA << ", offB = " << offB << ", offC = " <<
- offC << ::std::endl;
- ::std::cerr << "lda = " << lda << ", ldb = " << ldb << ", ldc = " <<
- ldc << ::std::endl;
+ ::std::cerr
+ << orderStr(order) << ", "
+ << transStr(transA) << ", "
+ << transStr(transB) << ", "
+ << "M = " << M << ", "
+ << "N = " << N << ", "
+ << "K = " << K << ", "
+ << "offA = " << offA << ", "
+ << "offB = " << offB << ", "
+ << "offC = " << offC << ", "
+ << "lda = " << lda << ", "
+ << "ldb = " << ldb << ", "
+ << "ldc = " << ldc;
if (useAlpha) {
- ::std::cerr << "alpha = (" << alpha.re << "," << alpha.imag
- << ")" << ::std::endl;
+ ::std::cerr << ", "
+ << "alpha = (" << alpha.re << "," << alpha.imag << ")";
}
if (useBeta) {
- ::std::cerr << "beta = (" << beta.re << "," << beta.imag
- << ")" << ::std::endl;
+ ::std::cerr << ", "
+ << "beta = (" << beta.re << "," << beta.imag << ")";
}
+ ::std::cerr << std::endl;
}
void
diff --git a/src/tests/correctness/corr-gemm.cpp b/src/tests/correctness/corr-gemm.cpp
index 5837bed..5d84983 100644
--- a/src/tests/correctness/corr-gemm.cpp
+++ b/src/tests/correctness/corr-gemm.cpp
@@ -104,14 +104,14 @@ gemmCorrectnessTest(TestParams *params)
beta = convertMultiplier<T>(params->beta);
}
- ::std::cerr << "Generating input data... ";
+ //::std::cerr << "Generating input data... ";
randomGemmMatrices<T>(params->order, params->transA, params->transB,
params->M, params->N, params->K, useAlpha, &alpha, A, params->lda,
B, params->ldb, useBeta, &beta, blasC, params->ldc);
memcpy(clblasC, blasC, params->rowsC * params->columnsC * sizeof(*blasC));
- ::std::cerr << "Done" << ::std::endl;
+ //::std::cerr << "Done" << ::std::endl;
- ::std::cerr << "Calling reference xGEMM routine... ";
+ //::std::cerr << "Calling reference xGEMM routine... ";
if (params->order == clblasColumnMajor) {
::clMath::blas::gemm(clblasColumnMajor, params->transA, params->transB,
params->M, params->N, params->K, alpha, A,
@@ -139,7 +139,7 @@ gemmCorrectnessTest(TestParams *params)
delete[] reorderedB;
delete[] reorderedA;
}
- ::std::cerr << "Done" << ::std::endl;
+ //::std::cerr << "Done" << ::std::endl;
bufA = base->createEnqueueBuffer(A, params->rowsA * params->columnsA *
sizeof(*A), params->offA * sizeof(*A),
@@ -167,7 +167,7 @@ gemmCorrectnessTest(TestParams *params)
return;
}
- ::std::cerr << "Calling clblas xGEMM routine... ";
+ //::std::cerr << "Calling clblas xGEMM routine... ";
err = (cl_int)::clMath::clblas::gemm(params->order, params->transA,
params->transB, params->M, params->N, params->K, alpha, bufA,
params->offA, params->lda, bufB, params->offBX, params->ldb, beta,
@@ -188,7 +188,7 @@ gemmCorrectnessTest(TestParams *params)
delete[] events;
ASSERT_EQ(CL_SUCCESS, err) << "waitForSuccessfulFinish()";
}
- ::std::cerr << "Done" << ::std::endl;
+ //::std::cerr << "Done" << ::std::endl;
clEnqueueReadBuffer(base->commandQueues()[0], bufC, CL_TRUE,
params->offCY * sizeof(*clblasC),
diff --git a/src/tests/include/gemm.h b/src/tests/include/gemm.h
index c580d45..1307fac 100644
--- a/src/tests/include/gemm.h
+++ b/src/tests/include/gemm.h
@@ -150,12 +150,12 @@ protected:
rowsC = ldc;
break;
}
-
+
+ ::std::cerr << " seed = " << seed << ", "
+ << "queues = " << numCommandQueues << ", ";
printTestParams(order, transA, transB, M, N, K, useAlpha,
base->alpha(), offA, lda, offB, ldb, useBeta,
base->beta(), offC, ldc);
- ::std::cerr << "seed = " << seed << ::std::endl;
- ::std::cerr << "queues = " << numCommandQueues << ::std::endl;
}
clblasOrder order;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/clblas.git
More information about the debian-science-commits
mailing list