[caffe-contrib] 240/362: Imported Upstream version 1.0.0~rc2-git20151212-g7953918
Zhou Mo
cdluminate-guest at moszumanska.debian.org
Tue May 3 09:24:37 UTC 2016
This is an automated email from the git hooks/post-receive script.
cdluminate-guest pushed a commit to branch master
in repository caffe-contrib.
commit 9cdffcc519df5adaeeb03f8bada642d6860f2ae8
Author: Zhou Mo <cdluminate at gmail.com>
Date: Sat Dec 12 07:06:39 2015 +0000
Imported Upstream version 1.0.0~rc2-git20151212-g7953918
---
CMakeLists.txt | 17 +-
INSTALL.md | 4 +-
Makefile | 16 +-
Makefile.config.example | 10 +-
README.md | 3 +
cmake/ConfigGen.cmake | 3 +
cmake/Cuda.cmake | 35 +-
cmake/Dependencies.cmake | 9 +-
cmake/Summary.cmake | 7 +-
cmake/Templates/caffe_config.h.in | 3 +-
docs/installation.md | 3 +-
docs/tutorial/forward_backward.md | 2 +-
docs/tutorial/interfaces.md | 2 +-
docs/tutorial/layers.md | 6 +-
docs/tutorial/solver.md | 28 +-
examples/00-classification.ipynb | 2 +-
.../cifar10/cifar10_full_sigmoid_solver.prototxt | 28 +
.../cifar10_full_sigmoid_solver_bn.prototxt | 28 +
.../cifar10_full_sigmoid_train_test.prototxt | 212 ++++++
.../cifar10_full_sigmoid_train_test_bn.prototxt | 240 ++++++
examples/cifar10/convert_cifar_data.cpp | 13 +-
examples/cifar10/train_full_sigmoid.sh | 7 +
examples/cifar10/train_full_sigmoid_bn.sh | 7 +
examples/cpp_classification/classification.cpp | 8 +-
examples/mnist/convert_mnist_data.cpp | 12 +-
examples/mnist/lenet_adadelta_solver.prototxt | 2 +-
examples/mnist/lenet_solver_adam.prototxt | 2 +-
examples/mnist/lenet_solver_rmsprop.prototxt | 2 +-
examples/mnist/lenet_stepearly_solver.prototxt | 28 -
.../mnist_autoencoder_solver_adadelta.prototxt | 2 +-
.../mnist_autoencoder_solver_adagrad.prototxt | 2 +-
.../mnist_autoencoder_solver_nesterov.prototxt | 2 +-
examples/siamese/convert_mnist_siamese_data.cpp | 7 +-
examples/web_demo/requirements.txt | 1 +
include/caffe/blob.hpp | 1 -
include/caffe/caffe.hpp | 3 +-
include/caffe/common_layers.hpp | 678 -----------------
include/caffe/data_layers.hpp | 344 ---------
include/caffe/filler.hpp | 1 -
include/caffe/layer.hpp | 2 +-
include/caffe/layer_factory.hpp | 1 +
include/caffe/layers/absval_layer.hpp | 68 ++
include/caffe/layers/accuracy_layer.hpp | 95 +++
include/caffe/layers/argmax_layer.hpp | 77 ++
include/caffe/layers/base_conv_layer.hpp | 168 +++++
include/caffe/layers/base_data_layer.hpp | 86 +++
include/caffe/layers/batch_norm_layer.hpp | 81 ++
include/caffe/layers/batch_reindex_layer.hpp | 83 +++
include/caffe/layers/bnll_layer.hpp | 70 ++
include/caffe/layers/concat_layer.hpp | 87 +++
include/caffe/layers/contrastive_loss_layer.hpp | 101 +++
include/caffe/layers/conv_layer.hpp | 81 ++
include/caffe/layers/cudnn_conv_layer.hpp | 72 ++
include/caffe/layers/cudnn_lcn_layer.hpp | 49 ++
include/caffe/layers/cudnn_lrn_layer.hpp | 44 ++
include/caffe/layers/cudnn_pooling_layer.hpp | 49 ++
include/caffe/layers/cudnn_relu_layer.hpp | 45 ++
include/caffe/layers/cudnn_sigmoid_layer.hpp | 45 ++
include/caffe/layers/cudnn_softmax_layer.hpp | 45 ++
include/caffe/layers/cudnn_tanh_layer.hpp | 45 ++
include/caffe/layers/data_layer.hpp | 39 +
include/caffe/layers/deconv_layer.hpp | 51 ++
include/caffe/layers/dropout_layer.hpp | 80 ++
include/caffe/layers/dummy_data_layer.hpp | 49 ++
include/caffe/layers/eltwise_layer.hpp | 51 ++
include/caffe/layers/embed_layer.hpp | 52 ++
include/caffe/layers/euclidean_loss_layer.hpp | 107 +++
include/caffe/layers/exp_layer.hpp | 80 ++
include/caffe/layers/filter_layer.hpp | 77 ++
include/caffe/layers/flatten_layer.hpp | 61 ++
include/caffe/layers/hdf5_data_layer.hpp | 62 ++
include/caffe/layers/hdf5_output_layer.hpp | 64 ++
include/caffe/layers/hinge_loss_layer.hpp | 104 +++
include/caffe/layers/im2col_layer.hpp | 63 ++
include/caffe/layers/image_data_layer.hpp | 47 ++
include/caffe/layers/infogain_loss_layer.hpp | 110 +++
include/caffe/layers/inner_product_layer.hpp | 51 ++
include/caffe/layers/log_layer.hpp | 82 ++
include/caffe/layers/loss_layer.hpp | 53 ++
include/caffe/layers/lrn_layer.hpp | 94 +++
include/caffe/layers/memory_data_layer.hpp | 63 ++
.../layers/multinomial_logistic_loss_layer.hpp | 92 +++
include/caffe/layers/mvn_layer.hpp | 48 ++
include/caffe/layers/neuron_layer.hpp | 32 +
include/caffe/layers/pooling_layer.hpp | 60 ++
include/caffe/layers/power_layer.hpp | 89 +++
include/caffe/layers/prelu_layer.hpp | 101 +++
include/caffe/{ => layers}/python_layer.hpp | 6 +
include/caffe/layers/reduction_layer.hpp | 59 ++
include/caffe/layers/relu_layer.hpp | 85 +++
include/caffe/layers/reshape_layer.hpp | 52 ++
.../layers/sigmoid_cross_entropy_loss_layer.hpp | 110 +++
include/caffe/layers/sigmoid_layer.hpp | 71 ++
include/caffe/layers/silence_layer.hpp | 43 ++
include/caffe/layers/slice_layer.hpp | 51 ++
include/caffe/layers/softmax_layer.hpp | 50 ++
include/caffe/layers/softmax_loss_layer.hpp | 130 ++++
include/caffe/layers/split_layer.hpp | 45 ++
include/caffe/layers/spp_layer.hpp | 76 ++
include/caffe/layers/tanh_layer.hpp | 73 ++
include/caffe/layers/threshold_layer.hpp | 64 ++
include/caffe/layers/tile_layer.hpp | 43 ++
include/caffe/layers/window_data_layer.hpp | 55 ++
include/caffe/loss_layers.hpp | 773 -------------------
include/caffe/neuron_layers.hpp | 809 --------------------
include/caffe/sgd_solvers.hpp | 148 ++++
include/caffe/solver.hpp | 173 +----
include/caffe/solver_factory.hpp | 137 ++++
include/caffe/syncedmem.hpp | 16 +-
include/caffe/test/test_gradient_check_util.hpp | 5 +-
include/caffe/util/blocking_queue.hpp | 2 -
include/caffe/util/cudnn.hpp | 3 +
include/caffe/util/device_alternate.hpp | 10 +-
include/caffe/util/format.hpp | 18 +
include/caffe/util/im2col.hpp | 4 +-
include/caffe/util/io.hpp | 57 +-
include/caffe/util/math_functions.hpp | 7 -
include/caffe/util/upgrade_proto.hpp | 26 +-
include/caffe/vision_layers.hpp | 589 ---------------
matlab/+caffe/+test/test_io.m | 18 +
matlab/+caffe/io.m | 8 +
matlab/+caffe/private/caffe_.cpp | 29 +-
matlab/+caffe/run_tests.m | 3 +-
matlab/hdf5creation/store2hdf5.m | 4 +-
models/bvlc_reference_caffenet/train_val.prototxt | 2 +-
models/finetune_flickr_style/train_val.prototxt | 14 +-
python/CMakeLists.txt | 2 +-
python/caffe/_caffe.cpp | 11 +-
python/caffe/draw.py | 6 +-
python/caffe/io.py | 23 +-
python/caffe/pycaffe.py | 6 +-
python/caffe/test/test_io.py | 41 +
python/caffe/test/test_python_layer.py | 2 +
.../caffe/test/test_python_layer_with_param_str.py | 2 +
python/caffe/test/test_solver.py | 11 +-
python/detect.py | 2 +-
scripts/download_model_binary.py | 2 +-
scripts/travis/travis_install.sh | 52 +-
src/caffe/data_reader.cpp | 2 +-
src/caffe/layer_factory.cpp | 64 +-
src/caffe/layers/absval_layer.cpp | 3 +-
src/caffe/layers/absval_layer.cu | 3 +-
src/caffe/layers/accuracy_layer.cpp | 5 +-
src/caffe/layers/argmax_layer.cpp | 82 +-
src/caffe/layers/base_conv_layer.cpp | 3 +-
src/caffe/layers/base_data_layer.cpp | 11 +-
src/caffe/layers/base_data_layer.cu | 2 +-
src/caffe/layers/batch_norm_layer.cpp | 239 ++++++
src/caffe/layers/batch_norm_layer.cu | 171 +++++
src/caffe/layers/batch_reindex_layer.cpp | 78 ++
src/caffe/layers/batch_reindex_layer.cu | 106 +++
src/caffe/layers/bnll_layer.cpp | 3 +-
src/caffe/layers/bnll_layer.cu | 3 +-
src/caffe/layers/concat_layer.cpp | 3 +-
src/caffe/layers/concat_layer.cu | 3 +-
src/caffe/layers/contrastive_loss_layer.cpp | 7 +-
src/caffe/layers/contrastive_loss_layer.cu | 4 +-
src/caffe/layers/conv_layer.cpp | 6 +-
src/caffe/layers/conv_layer.cu | 6 +-
src/caffe/layers/cudnn_conv_layer.cpp | 144 +++-
src/caffe/layers/cudnn_conv_layer.cu | 70 +-
src/caffe/layers/cudnn_lcn_layer.cpp | 73 ++
.../{cudnn_pooling_layer.cu => cudnn_lcn_layer.cu} | 33 +-
src/caffe/layers/cudnn_lrn_layer.cpp | 53 ++
.../{cudnn_pooling_layer.cu => cudnn_lrn_layer.cu} | 33 +-
src/caffe/layers/cudnn_pooling_layer.cpp | 6 +-
src/caffe/layers/cudnn_pooling_layer.cu | 6 +-
src/caffe/layers/cudnn_relu_layer.cpp | 4 +-
src/caffe/layers/cudnn_relu_layer.cu | 4 +-
src/caffe/layers/cudnn_sigmoid_layer.cpp | 4 +-
src/caffe/layers/cudnn_sigmoid_layer.cu | 4 +-
src/caffe/layers/cudnn_softmax_layer.cpp | 6 +-
src/caffe/layers/cudnn_softmax_layer.cu | 6 +-
src/caffe/layers/cudnn_tanh_layer.cpp | 4 +-
src/caffe/layers/cudnn_tanh_layer.cu | 4 +-
src/caffe/layers/data_layer.cpp | 8 +-
src/caffe/layers/deconv_layer.cpp | 6 +-
src/caffe/layers/deconv_layer.cu | 6 +-
src/caffe/layers/dropout_layer.cpp | 5 +-
src/caffe/layers/dropout_layer.cu | 9 +-
src/caffe/layers/dummy_data_layer.cpp | 3 +-
src/caffe/layers/eltwise_layer.cpp | 3 +-
src/caffe/layers/eltwise_layer.cu | 3 +-
src/caffe/layers/embed_layer.cpp | 5 +-
src/caffe/layers/embed_layer.cu | 5 +-
src/caffe/layers/euclidean_loss_layer.cpp | 4 +-
src/caffe/layers/euclidean_loss_layer.cu | 4 +-
src/caffe/layers/exp_layer.cpp | 4 +-
src/caffe/layers/exp_layer.cu | 4 +-
src/caffe/layers/filter_layer.cpp | 4 +-
src/caffe/layers/filter_layer.cu | 3 +-
src/caffe/layers/flatten_layer.cpp | 4 +-
src/caffe/layers/hdf5_data_layer.cpp | 3 +-
src/caffe/layers/hdf5_data_layer.cu | 5 +-
src/caffe/layers/hdf5_output_layer.cpp | 5 +-
src/caffe/layers/hdf5_output_layer.cu | 5 +-
src/caffe/layers/hinge_loss_layer.cpp | 6 +-
src/caffe/layers/im2col_layer.cpp | 4 +-
src/caffe/layers/im2col_layer.cu | 4 +-
src/caffe/layers/image_data_layer.cpp | 5 +-
src/caffe/layers/infogain_loss_layer.cpp | 5 +-
src/caffe/layers/inner_product_layer.cpp | 5 +-
src/caffe/layers/inner_product_layer.cu | 5 +-
src/caffe/layers/log_layer.cpp | 4 +-
src/caffe/layers/log_layer.cu | 4 +-
src/caffe/layers/loss_layer.cpp | 8 +-
src/caffe/layers/lrn_layer.cpp | 4 +-
src/caffe/layers/lrn_layer.cu | 3 +-
src/caffe/layers/memory_data_layer.cpp | 4 +-
.../layers/multinomial_logistic_loss_layer.cpp | 5 +-
src/caffe/layers/mvn_layer.cpp | 46 +-
src/caffe/layers/mvn_layer.cu | 47 +-
src/caffe/layers/neuron_layer.cpp | 3 +-
src/caffe/layers/pooling_layer.cpp | 5 +-
src/caffe/layers/pooling_layer.cu | 3 +-
src/caffe/layers/power_layer.cpp | 4 +-
src/caffe/layers/power_layer.cu | 4 +-
src/caffe/layers/prelu_layer.cpp | 5 +-
src/caffe/layers/prelu_layer.cu | 48 +-
src/caffe/layers/reduction_layer.cpp | 5 +-
src/caffe/layers/reduction_layer.cu | 4 +-
src/caffe/layers/relu_layer.cpp | 3 +-
src/caffe/layers/relu_layer.cu | 3 +-
src/caffe/layers/reshape_layer.cpp | 3 +-
.../layers/sigmoid_cross_entropy_loss_layer.cpp | 5 +-
.../layers/sigmoid_cross_entropy_loss_layer.cu | 5 +-
src/caffe/layers/sigmoid_layer.cpp | 4 +-
src/caffe/layers/sigmoid_layer.cu | 4 +-
src/caffe/layers/silence_layer.cpp | 5 +-
src/caffe/layers/silence_layer.cu | 5 +-
src/caffe/layers/slice_layer.cpp | 3 +-
src/caffe/layers/slice_layer.cu | 3 +-
src/caffe/layers/softmax_layer.cpp | 3 +-
src/caffe/layers/softmax_layer.cu | 3 +-
src/caffe/layers/softmax_loss_layer.cpp | 58 +-
src/caffe/layers/softmax_loss_layer.cu | 35 +-
src/caffe/layers/split_layer.cpp | 3 +-
src/caffe/layers/split_layer.cu | 3 +-
src/caffe/layers/spp_layer.cpp | 11 +-
src/caffe/layers/tanh_layer.cpp | 4 +-
src/caffe/layers/tanh_layer.cu | 4 +-
src/caffe/layers/threshold_layer.cpp | 4 +-
src/caffe/layers/threshold_layer.cu | 4 +-
src/caffe/layers/tile_layer.cpp | 3 +-
src/caffe/layers/tile_layer.cu | 3 +-
src/caffe/layers/window_data_layer.cpp | 7 +-
src/caffe/net.cpp | 188 ++---
src/caffe/parallel.cpp | 4 -
src/caffe/proto/caffe.proto | 77 +-
src/caffe/solver.cpp | 821 +--------------------
src/caffe/solvers/adadelta_solver.cpp | 156 ++++
src/caffe/solvers/adagrad_solver.cpp | 89 +++
src/caffe/solvers/adam_solver.cpp | 113 +++
src/caffe/solvers/nesterov_solver.cpp | 71 ++
src/caffe/solvers/rmsprop_solver.cpp | 85 +++
src/caffe/solvers/sgd_solver.cpp | 348 +++++++++
src/caffe/syncedmem.cpp | 12 +-
src/caffe/test/test_accuracy_layer.cpp | 4 +-
src/caffe/test/test_argmax_layer.cpp | 139 +++-
src/caffe/test/test_batch_norm_layer.cpp | 133 ++++
src/caffe/test/test_batch_reindex_layer.cpp | 118 +++
src/caffe/test/test_benchmark.cpp | 6 +-
src/caffe/test/test_blob.cpp | 1 -
src/caffe/test/test_common.cpp | 2 -
src/caffe/test/test_concat_layer.cpp | 3 +-
src/caffe/test/test_contrastive_loss_layer.cpp | 6 +-
src/caffe/test/test_convolution_layer.cpp | 7 +-
src/caffe/test/test_data_layer.cpp | 2 +-
src/caffe/test/test_deconvolution_layer.cpp | 3 +-
src/caffe/test/test_dummy_data_layer.cpp | 2 +-
src/caffe/test/test_eltwise_layer.cpp | 2 +-
src/caffe/test/test_embed_layer.cpp | 3 +-
src/caffe/test/test_euclidean_loss_layer.cpp | 4 +-
src/caffe/test/test_filler.cpp | 2 -
src/caffe/test/test_filter_layer.cpp | 4 +-
src/caffe/test/test_flatten_layer.cpp | 3 +-
src/caffe/test/test_gradient_based_solver.cpp | 56 +-
src/caffe/test/test_hdf5_output_layer.cpp | 2 +-
src/caffe/test/test_hdf5data_layer.cpp | 5 +-
src/caffe/test/test_hinge_loss_layer.cpp | 4 +-
src/caffe/test/test_im2col_kernel.cu | 3 +-
src/caffe/test/test_im2col_layer.cpp | 3 +-
src/caffe/test/test_image_data_layer.cpp | 2 +-
src/caffe/test/test_infogain_loss_layer.cpp | 5 +-
src/caffe/test/test_inner_product_layer.cpp | 3 +-
src/caffe/test/test_lrn_layer.cpp | 204 ++++-
src/caffe/test/test_math_functions.cpp | 43 --
src/caffe/test/test_maxpool_dropout_layers.cpp | 4 +-
src/caffe/test/test_memory_data_layer.cpp | 2 +-
.../test/test_multinomial_logistic_loss_layer.cpp | 5 +-
src/caffe/test/test_mvn_layer.cpp | 4 +-
src/caffe/test/test_neuron_layer.cpp | 21 +-
src/caffe/test/test_pooling_layer.cpp | 7 +-
src/caffe/test/test_power_layer.cpp | 2 +-
src/caffe/test/test_random_number_generator.cpp | 1 -
src/caffe/test/test_reduction_layer.cpp | 3 +-
src/caffe/test/test_reshape_layer.cpp | 3 +-
.../test/test_sigmoid_cross_entropy_loss_layer.cpp | 4 +-
src/caffe/test/test_slice_layer.cpp | 3 +-
src/caffe/test/test_softmax_layer.cpp | 7 +-
src/caffe/test/test_softmax_with_loss_layer.cpp | 4 +-
src/caffe/test/test_solver.cpp | 1 +
src/caffe/test/test_solver_factory.cpp | 50 ++
src/caffe/test/test_split_layer.cpp | 3 +-
src/caffe/test/test_spp_layer.cpp | 9 +-
src/caffe/test/test_stochastic_pooling.cpp | 3 +-
src/caffe/test/test_syncedmem.cpp | 1 -
src/caffe/test/test_tanh_layer.cpp | 2 +-
src/caffe/test/test_threshold_layer.cpp | 2 +-
src/caffe/test/test_tile_layer.cpp | 3 +-
src/caffe/test/test_upgrade_proto.cpp | 64 +-
src/caffe/test/test_util_blas.cpp | 2 -
src/caffe/util/blocking_queue.cpp | 2 +-
src/caffe/util/db.cpp | 2 +
src/caffe/util/db_lmdb.cpp | 17 +-
src/caffe/util/im2col.cpp | 81 +-
src/caffe/util/im2col.cu | 78 +-
src/caffe/util/io.cpp | 2 +-
src/caffe/util/math_functions.cpp | 22 -
src/caffe/util/math_functions.cu | 47 --
src/caffe/util/upgrade_proto.cpp | 188 +++--
tools/caffe.cpp | 4 +-
tools/convert_imageset.cpp | 8 +-
tools/extra/parse_log.sh | 7 +-
tools/extra/plot_training_log.py.example | 2 +-
tools/extra/summarize.py | 140 ++++
tools/extract_features.cpp | 29 +-
tools/upgrade_solver_proto_text.cpp | 50 ++
328 files changed, 8989 insertions(+), 5486 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 37f937f..c446c60 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,10 @@
cmake_minimum_required(VERSION 2.8.7)
+if(POLICY CMP0046)
+ cmake_policy(SET CMP0046 NEW)
+endif()
+if(POLICY CMP0054)
+ cmake_policy(SET CMP0054 NEW)
+endif()
# ---[ Caffe project
project(Caffe C CXX)
@@ -23,9 +29,10 @@ set(python_version "2" CACHE STRING "Specify which Python version to use")
caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE)
caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE)
caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON)
-caffe_option(USE_LMDB "Build with lmdb" ON)
-caffe_option(USE_LEVELDB "Build with levelDB" ON)
caffe_option(USE_OPENCV "Build with OpenCV support" ON)
+caffe_option(USE_LEVELDB "Build with levelDB" ON)
+caffe_option(USE_LMDB "Build with lmdb" ON)
+caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF)
# ---[ Dependencies
include(cmake/Dependencies.cmake)
@@ -66,8 +73,10 @@ add_subdirectory(docs)
add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake)
# ---[ pytest target
-add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python )
-add_dependencies(pytest pycaffe)
+if(BUILD_python)
+ add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python )
+ add_dependencies(pytest pycaffe)
+endif()
# ---[ Configuration summary
caffe_print_configuration_summary()
diff --git a/INSTALL.md b/INSTALL.md
index 42fcf02..05c714d 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -3,5 +3,5 @@
See http://caffe.berkeleyvision.org/installation.html for the latest
installation instructions.
-Check the issue tracker in case you need help:
-https://github.com/BVLC/caffe/issues
+Check the users group in case you need help:
+https://groups.google.com/forum/#!forum/caffe-users
diff --git a/Makefile b/Makefile
index 5fb6394..985fffd 100644
--- a/Makefile
+++ b/Makefile
@@ -78,7 +78,7 @@ NONEMPTY_LINT_REPORT := $(BUILD_DIR)/$(LINT_EXT)
# PY$(PROJECT)_SRC is the python wrapper for $(PROJECT)
PY$(PROJECT)_SRC := python/$(PROJECT)/_$(PROJECT).cpp
PY$(PROJECT)_SO := python/$(PROJECT)/_$(PROJECT).so
-PY$(PROJECT)_HXX := include/$(PROJECT)/python_layer.hpp
+PY$(PROJECT)_HXX := include/$(PROJECT)/layers/python_layer.hpp
# MAT$(PROJECT)_SRC is the mex entrance point of matlab package for $(PROJECT)
MAT$(PROJECT)_SRC := matlab/+$(PROJECT)/private/$(PROJECT)_.cpp
ifneq ($(MATLAB_DIR),)
@@ -170,7 +170,7 @@ ifneq ($(CPU_ONLY), 1)
LIBRARIES := cudart cublas curand
endif
-LIBRARIES += glog gflags protobuf boost_system m hdf5_hl hdf5
+LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_hl hdf5
# handle IO dependencies
USE_LEVELDB ?= 1
@@ -184,7 +184,12 @@ ifeq ($(USE_LMDB), 1)
LIBRARIES += lmdb
endif
ifeq ($(USE_OPENCV), 1)
- LIBRARIES += opencv_core opencv_highgui opencv_imgproc
+ LIBRARIES += opencv_core opencv_highgui opencv_imgproc
+
+ ifeq ($(OPENCV_VERSION), 3)
+ LIBRARIES += opencv_imgcodecs
+ endif
+
endif
PYTHON_LIBRARIES := boost_python python2.7
WARNINGS := -Wall -Wno-sign-compare
@@ -313,6 +318,9 @@ ifeq ($(USE_LEVELDB), 1)
endif
ifeq ($(USE_LMDB), 1)
COMMON_FLAGS += -DUSE_LMDB
+ifeq ($(ALLOW_LMDB_NOLOCK), 1)
+ COMMON_FLAGS += -DALLOW_LMDB_NOLOCK
+endif
endif
# CPU-only configuration
@@ -652,7 +660,7 @@ $(DISTRIBUTE_DIR): all py | $(DISTRIBUTE_SUBDIRS)
cp $(EXAMPLE_BINS) $(DISTRIBUTE_DIR)/bin
# add libraries
cp $(STATIC_NAME) $(DISTRIBUTE_DIR)/lib
- cp $(DYNAMIC_NAME) $(DISTRIBUTE_DIR)/lib
+ install -m 644 $(DYNAMIC_NAME) $(DISTRIBUTE_DIR)/lib
# add python - it's not the standard way, indeed...
cp -r python $(DISTRIBUTE_DIR)/python
diff --git a/Makefile.config.example b/Makefile.config.example
index a20bad2..1dd6a8f 100644
--- a/Makefile.config.example
+++ b/Makefile.config.example
@@ -8,9 +8,17 @@
# CPU_ONLY := 1
# uncomment to disable IO dependencies and corresponding data layers
+# USE_OPENCV := 0
# USE_LEVELDB := 0
# USE_LMDB := 0
-# USE_OPENCV := 0
+
+# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary)
+# You should not set this flag if you will be reading LMDBs with any
+# possibility of simultaneous read and write
+# ALLOW_LMDB_NOLOCK := 1
+
+# Uncomment if you're using OpenCV 3
+# OPENCV_VERSION := 3
# To customize your choice of compiler, uncomment and set the following.
# N.B. the default for Linux is g++ and the default for OSX is clang++
diff --git a/README.md b/README.md
index ebec286..44b9e62 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,8 @@
# Caffe
+[](https://travis-ci.org/BVLC/caffe)
+[](LICENSE)
+
Caffe is a deep learning framework made with expression, speed, and modularity in mind.
It is developed by the Berkeley Vision and Learning Center ([BVLC](http://bvlc.eecs.berkeley.edu)) and community contributors.
diff --git a/cmake/ConfigGen.cmake b/cmake/ConfigGen.cmake
index 8b25996..0563711 100644
--- a/cmake/ConfigGen.cmake
+++ b/cmake/ConfigGen.cmake
@@ -62,6 +62,9 @@ function(caffe_generate_export_configs)
if(USE_LMDB)
list(APPEND Caffe_DEFINITIONS -DUSE_LMDB)
+ if (ALLOW_LMDB_NOLOCK)
+ list(APPEND Caffe_DEFINITIONS -DALLOW_LMDB_NOLOCK)
+ endif()
endif()
if(USE_LEVELDB)
diff --git a/cmake/Cuda.cmake b/cmake/Cuda.cmake
index ff58d31..286a428 100644
--- a/cmake/Cuda.cmake
+++ b/cmake/Cuda.cmake
@@ -132,7 +132,7 @@ function(caffe_select_nvcc_arch_flags out_variable)
endfunction()
################################################################################################
-# Short command for cuda comnpilation
+# Short command for cuda compilation
# Usage:
# caffe_cuda_compile(<objlist_variable> <cuda_files>)
macro(caffe_cuda_compile objlist_variable)
@@ -183,12 +183,41 @@ function(detect_cuDNN)
set(HAVE_CUDNN TRUE PARENT_SCOPE)
set(CUDNN_FOUND TRUE PARENT_SCOPE)
+ file(READ ${CUDNN_INCLUDE}/cudnn.h CUDNN_VERSION_FILE_CONTENTS)
+
+ # cuDNN v3 and beyond
+ string(REGEX MATCH "define CUDNN_MAJOR * +([0-9]+)"
+ CUDNN_VERSION_MAJOR "${CUDNN_VERSION_FILE_CONTENTS}")
+ string(REGEX REPLACE "define CUDNN_MAJOR * +([0-9]+)" "\\1"
+ CUDNN_VERSION_MAJOR "${CUDNN_VERSION_MAJOR}")
+ string(REGEX MATCH "define CUDNN_MINOR * +([0-9]+)"
+ CUDNN_VERSION_MINOR "${CUDNN_VERSION_FILE_CONTENTS}")
+ string(REGEX REPLACE "define CUDNN_MINOR * +([0-9]+)" "\\1"
+ CUDNN_VERSION_MINOR "${CUDNN_VERSION_MINOR}")
+ string(REGEX MATCH "define CUDNN_PATCHLEVEL * +([0-9]+)"
+ CUDNN_VERSION_PATCH "${CUDNN_VERSION_FILE_CONTENTS}")
+ string(REGEX REPLACE "define CUDNN_PATCHLEVEL * +([0-9]+)" "\\1"
+ CUDNN_VERSION_PATCH "${CUDNN_VERSION_PATCH}")
+
+ if(NOT CUDNN_VERSION_MAJOR)
+ set(CUDNN_VERSION "???")
+ else()
+ set(CUDNN_VERSION "${CUDNN_VERSION_MAJOR}.${CUDNN_VERSION_MINOR}.${CUDNN_VERSION_PATCH}")
+ endif()
+
+ message(STATUS "Found cuDNN: ver. ${CUDNN_VERSION} found (include: ${CUDNN_INCLUDE}, library: ${CUDNN_LIBRARY})")
+
+ string(COMPARE LESS "${CUDNN_VERSION_MAJOR}" 3 cuDNNVersionIncompatible)
+ if(cuDNNVersionIncompatible)
+ message(FATAL_ERROR "cuDNN version >3 is required.")
+ endif()
+
+ set(CUDNN_VERSION "${CUDNN_VERSION}" PARENT_SCOPE)
mark_as_advanced(CUDNN_INCLUDE CUDNN_LIBRARY CUDNN_ROOT)
- message(STATUS "Found cuDNN (include: ${CUDNN_INCLUDE}, library: ${CUDNN_LIBRARY})")
+
endif()
endfunction()
-
################################################################################################
### Non macro section
################################################################################################
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index d68d7bf..51a803c 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -2,7 +2,7 @@
set(Caffe_LINKER_LIBS "")
# ---[ Boost
-find_package(Boost 1.46 REQUIRED COMPONENTS system thread)
+find_package(Boost 1.46 REQUIRED COMPONENTS system thread filesystem)
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${Boost_LIBRARIES})
@@ -34,6 +34,9 @@ if(USE_LMDB)
include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES})
add_definitions(-DUSE_LMDB)
+ if(ALLOW_LMDB_NOLOCK)
+ add_definitions(-DALLOW_LMDB_NOLOCK)
+ endif()
endif()
# ---[ LevelDB
@@ -55,9 +58,9 @@ endif()
include(cmake/Cuda.cmake)
if(NOT HAVE_CUDA)
if(CPU_ONLY)
- message("-- CUDA is disabled. Building without it...")
+ message(STATUS "-- CUDA is disabled. Building without it...")
else()
- message("-- CUDA is not detected by cmake. Building without it...")
+ message(WARNING "-- CUDA is not detected by cmake. Building without it...")
endif()
# TODO: remove this not cross platform define in future. Use caffe_config.h instead.
diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake
index 3d12e81..557a6f0 100644
--- a/cmake/Summary.cmake
+++ b/cmake/Summary.cmake
@@ -114,9 +114,10 @@ function(caffe_print_configuration_summary)
caffe_status(" BUILD_matlab : ${BUILD_matlab}")
caffe_status(" BUILD_docs : ${BUILD_docs}")
caffe_status(" CPU_ONLY : ${CPU_ONLY}")
- caffe_status(" USE_LMDB : ${USE_LMDB}")
- caffe_status(" USE_LEVELDB : ${USE_LEVELDB}")
caffe_status(" USE_OPENCV : ${USE_OPENCV}")
+ caffe_status(" USE_LEVELDB : ${USE_LEVELDB}")
+ caffe_status(" USE_LMDB : ${USE_LMDB}")
+ caffe_status(" ALLOW_LMDB_NOLOCK : ${ALLOW_LMDB_NOLOCK}")
caffe_status("")
caffe_status("Dependencies:")
caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")
@@ -141,7 +142,7 @@ function(caffe_print_configuration_summary)
caffe_status(" Target GPU(s) : ${CUDA_ARCH_NAME}" )
caffe_status(" GPU arch(s) : ${NVCC_FLAGS_EXTRA_readable}")
if(USE_CUDNN)
- caffe_status(" cuDNN : " HAVE_CUDNN THEN "Yes" ELSE "Not found")
+ caffe_status(" cuDNN : " HAVE_CUDNN THEN "Yes (ver. ${CUDNN_VERSION})" ELSE "Not found")
else()
caffe_status(" cuDNN : Disabled")
endif()
diff --git a/cmake/Templates/caffe_config.h.in b/cmake/Templates/caffe_config.h.in
index 9302022..8a31b43 100644
--- a/cmake/Templates/caffe_config.h.in
+++ b/cmake/Templates/caffe_config.h.in
@@ -33,5 +33,6 @@
/* IO libraries */
#cmakedefine USE_OPENCV
-#cmakedefine USE_LMDB
#cmakedefine USE_LEVELDB
+#cmakedefine USE_LMDB
+#cmakedefine ALLOW_LMDB_NOLOCK
diff --git a/docs/installation.md b/docs/installation.md
index 89a8c71..cce7ec3 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -30,13 +30,14 @@ Optional dependencies:
* [OpenCV](http://opencv.org/) >= 2.4 including 3.0
* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`)
+* cuDNN for GPU acceleration (v3)
Pycaffe and Matcaffe interfaces have their own natural needs.
* For Python Caffe: `Python 2.7` or `Python 3.3+`, `numpy (>= 1.7)`, boost-provided `boost.python`
* For MATLAB Caffe: MATLAB with the `mex` compiler.
-**cuDNN Caffe**: for fastest operation Caffe is accelerated by drop-in integration of [NVIDIA cuDNN](https://developer.nvidia.com/cudnn). To speed up your Caffe models, install cuDNN then uncomment the `USE_CUDNN := 1` flag in `Makefile.config` when installing Caffe. Acceleration is automatic. For now cuDNN v1 is integrated but see [PR #1731](https://github.com/BVLC/caffe/pull/1731) for v2.
+**cuDNN Caffe**: for fastest operation Caffe is accelerated by drop-in integration of [NVIDIA cuDNN](https://developer.nvidia.com/cudnn). To speed up your Caffe models, install cuDNN then uncomment the `USE_CUDNN := 1` flag in `Makefile.config` when installing Caffe. Acceleration is automatic. The current version is cuDNN v3; older versions are supported in older Caffe.
**CPU-only Caffe**: for cold-brewed CPU-only Caffe uncomment the `CPU_ONLY := 1` flag in `Makefile.config` to configure and build Caffe without CUDA. This is helpful for cloud or cluster deployment.
diff --git a/docs/tutorial/forward_backward.md b/docs/tutorial/forward_backward.md
index a645f00..528b993 100644
--- a/docs/tutorial/forward_backward.md
+++ b/docs/tutorial/forward_backward.md
@@ -29,7 +29,7 @@ The backward pass begins with the loss and computes the gradient with respect to
These computations follow immediately from defining the model: Caffe plans and carries out the forward and backward passes for you.
- The `Net::Forward()` and `Net::Backward()` methods carry out the respective passes while `Layer::Forward()` and `Layer::Backward()` compute each step.
-- Every layer type has `forward_{cpu,gpu}()` and `backward_{cpu,gpu}` methods to compute its steps according to the mode of computation. A layer may only implement CPU or GPU mode due to constraints or convenience.
+- Every layer type has `forward_{cpu,gpu}()` and `backward_{cpu,gpu}()` methods to compute its steps according to the mode of computation. A layer may only implement CPU or GPU mode due to constraints or convenience.
The [Solver](solver.html) optimizes a model by first calling forward to yield the output and loss, then calling backward to generate the gradient of the model, and then incorporating the gradient into a weight update that attempts to minimize the loss. Division of labor between the Solver, Net, and Layer keep Caffe modular and open to development.
diff --git a/docs/tutorial/interfaces.md b/docs/tutorial/interfaces.md
index 9006179..d7ff378 100644
--- a/docs/tutorial/interfaces.md
+++ b/docs/tutorial/interfaces.md
@@ -61,7 +61,7 @@ For a full example of fine-tuning, see examples/finetuning_on_flickr_style, but
The Python interface -- pycaffe -- is the `caffe` module and its scripts in caffe/python. `import caffe` to load models, do forward and backward, handle IO, visualize networks, and even instrument model solving. All model data, derivatives, and parameters are exposed for reading and writing.
-- `caffe.Net` is the central interface for loading, configuring, and running models. `caffe.Classsifier` and `caffe.Detector` provide convenience interfaces for common tasks.
+- `caffe.Net` is the central interface for loading, configuring, and running models. `caffe.Classifier` and `caffe.Detector` provide convenience interfaces for common tasks.
- `caffe.SGDSolver` exposes the solving interface.
- `caffe.io` handles input / output with preprocessing and protocol buffers.
- `caffe.draw` visualizes network architectures.
diff --git a/docs/tutorial/layers.md b/docs/tutorial/layers.md
index eabc792..7362aac 100644
--- a/docs/tutorial/layers.md
+++ b/docs/tutorial/layers.md
@@ -39,7 +39,7 @@ In contrast, other layers (with few exceptions) ignore the spatial structure of
- `n * c_i * h_i * w_i`
* Output
- `n * c_o * h_o * w_o`, where `h_o = (h_i + 2 * pad_h - kernel_h) / stride_h + 1` and `w_o` likewise.
-* Sample (as seen in `./examples/imagenet/imagenet_train_val.prototxt`)
+* Sample (as seen in `./models/bvlc_reference_caffenet/train_val.prototxt`)
layer {
name: "conv1"
@@ -83,7 +83,7 @@ The `Convolution` layer convolves the input image with a set of learnable filter
- `n * c * h_i * w_i`
* Output
- `n * c * h_o * w_o`, where h_o and w_o are computed in the same way as convolution.
-* Sample (as seen in `./examples/imagenet/imagenet_train_val.prototxt`)
+* Sample (as seen in `./models/bvlc_reference_caffenet/train_val.prototxt`)
layer {
name: "pool1"
@@ -197,7 +197,7 @@ In general, activation / Neuron layers are element-wise operators, taking one bo
* Parameters (`ReLUParameter relu_param`)
- Optional
- `negative_slope` [default 0]: specifies whether to leak the negative part by multiplying it with the slope value rather than setting it to 0.
-* Sample (as seen in `./examples/imagenet/imagenet_train_val.prototxt`)
+* Sample (as seen in `./models/bvlc_reference_caffenet/train_val.prototxt`)
layer {
name: "relu1"
diff --git a/docs/tutorial/solver.md b/docs/tutorial/solver.md
index b150f64..b719f71 100644
--- a/docs/tutorial/solver.md
+++ b/docs/tutorial/solver.md
@@ -8,12 +8,12 @@ The responsibilities of learning are divided between the Solver for overseeing t
The Caffe solvers are:
-- Stochastic Gradient Descent (`SGD`),
-- AdaDelta (`ADADELTA`),
-- Adaptive Gradient (`ADAGRAD`),
-- Adam (`ADAM`),
-- Nesterov's Accelerated Gradient (`NESTEROV`) and
-- RMSprop (`RMSPROP`)
+- Stochastic Gradient Descent (`type: "SGD"`),
+- AdaDelta (`type: "AdaDelta"`),
+- Adaptive Gradient (`type: "AdaGrad"`),
+- Adam (`type: "Adam"`),
+- Nesterov's Accelerated Gradient (`type: "Nesterov"`) and
+- RMSprop (`type: "RMSProp"`)
The solver
@@ -51,7 +51,7 @@ The parameter update $$\Delta W$$ is formed by the solver from the error gradien
### SGD
-**Stochastic gradient descent** (`solver_type: SGD`) updates the weights $$ W $$ by a linear combination of the negative gradient $$ \nabla L(W) $$ and the previous weight update $$ V_t $$.
+**Stochastic gradient descent** (`type: "SGD"`) updates the weights $$ W $$ by a linear combination of the negative gradient $$ \nabla L(W) $$ and the previous weight update $$ V_t $$.
The **learning rate** $$ \alpha $$ is the weight of the negative gradient.
The **momentum** $$ \mu $$ is the weight of the previous update.
@@ -113,7 +113,7 @@ If learning diverges (e.g., you start to see very large or `NaN` or `inf` loss v
### AdaDelta
-The **AdaDelta** (`solver_type: ADADELTA`) method (M. Zeiler [1]) is a "robust learning rate method". It is a gradient-based optimization method (like SGD). The update formulas are
+The **AdaDelta** (`type: "AdaDelta"`) method (M. Zeiler [1]) is a "robust learning rate method". It is a gradient-based optimization method (like SGD). The update formulas are
$$
\begin{align}
@@ -125,7 +125,7 @@ E[g^2]_t &= \delta{E[g^2]_{t-1} } + (1-\delta)g_{t}^2
\end{align}
$$
-and
+and
$$
(W_{t+1})_i =
@@ -139,7 +139,7 @@ $$
### AdaGrad
-The **adaptive gradient** (`solver_type: ADAGRAD`) method (Duchi et al. [1]) is a gradient-based optimization method (like SGD) that attempts to "find needles in haystacks in the form of very predictive but rarely seen features," in Duchi et al.'s words.
+The **adaptive gradient** (`type: "AdaGrad"`) method (Duchi et al. [1]) is a gradient-based optimization method (like SGD) that attempts to "find needles in haystacks in the form of very predictive but rarely seen features," in Duchi et al.'s words.
Given the update information from all previous iterations $$ \left( \nabla L(W) \right)_{t'} $$ for $$ t' \in \{1, 2, ..., t\} $$,
the update formulas proposed by [1] are as follows, specified for each component $$i$$ of the weights $$W$$:
@@ -159,7 +159,7 @@ Note that in practice, for weights $$ W \in \mathcal{R}^d $$, AdaGrad implementa
### Adam
-The **Adam** (`solver_type: ADAM`), proposed in Kingma et al. [1], is a gradient-based optimization method (like SGD). This includes an "adaptive moment estimation" ($$m_t, v_t$$) and can be regarded as a generalization of AdaGrad. The update formulas are
+The **Adam** (`type: "Adam"`), proposed in Kingma et al. [1], is a gradient-based optimization method (like SGD). This includes an "adaptive moment estimation" ($$m_t, v_t$$) and can be regarded as a generalization of AdaGrad. The update formulas are
$$
(m_t)_i = \beta_1 (m_{t-1})_i + (1-\beta_1)(\nabla L(W_t))_i,\\
@@ -181,7 +181,7 @@ Kingma et al. [1] proposed to use $$\beta_1 = 0.9, \beta_2 = 0.999, \varepsilon
### NAG
-**Nesterov's accelerated gradient** (`solver_type: NESTEROV`) was proposed by Nesterov [1] as an "optimal" method of convex optimization, achieving a convergence rate of $$ \mathcal{O}(1/t^2) $$ rather than the $$ \mathcal{O}(1/t) $$.
+**Nesterov's accelerated gradient** (`type: "Nesterov"`) was proposed by Nesterov [1] as an "optimal" method of convex optimization, achieving a convergence rate of $$ \mathcal{O}(1/t^2) $$ rather than the $$ \mathcal{O}(1/t) $$.
Though the required assumptions to achieve the $$ \mathcal{O}(1/t^2) $$ convergence typically will not hold for deep networks trained with Caffe (e.g., due to non-smoothness and non-convexity), in practice NAG can be a very effective method for optimizing certain types of deep learning architectures, as demonstrated for deep MNIST autoencoders by Sutskever et al. [2].
The weight update formulas look very similar to the SGD updates given above:
@@ -206,10 +206,10 @@ What distinguishes the method from SGD is the weight setting $$ W $$ on which we
### RMSprop
-The **RMSprop** (`solver_type: RMSPROP`), suggested by Tieleman in a Coursera course lecture, is a gradient-based optimization method (like SGD). The update formulas are
+The **RMSprop** (`type: "RMSProp"`), suggested by Tieleman in a Coursera course lecture, is a gradient-based optimization method (like SGD). The update formulas are
$$
-(v_t)_i =
+(v_t)_i =
\begin{cases}
(v_{t-1})_i + \delta, &(\nabla L(W_t))_i(\nabla L(W_{t-1}))_i > 0\\
(v_{t-1})_i \cdot (1-\delta), & \text{else}
diff --git a/examples/00-classification.ipynb b/examples/00-classification.ipynb
index 46bbb19..89b7dd3 100644
--- a/examples/00-classification.ipynb
+++ b/examples/00-classification.ipynb
@@ -119,7 +119,7 @@
"source": [
"net.blobs['data'].data[...] = transformer.preprocess('data', caffe.io.load_image(caffe_root + 'examples/images/cat.jpg'))\n",
"out = net.forward()\n",
- "print(\"Predicted class is #{}.\".format(out['prob'].argmax()))"
+ "print(\"Predicted class is #{}.\".format(out['prob'][0].argmax()))"
]
},
{
diff --git a/examples/cifar10/cifar10_full_sigmoid_solver.prototxt b/examples/cifar10/cifar10_full_sigmoid_solver.prototxt
new file mode 100644
index 0000000..7dd3ecb
--- /dev/null
+++ b/examples/cifar10/cifar10_full_sigmoid_solver.prototxt
@@ -0,0 +1,28 @@
+# reduce learning rate after 120 epochs (60000 iters) by factor 0f 10
+# then another factor of 10 after 10 more epochs (5000 iters)
+
+# The train/test net protocol buffer definition
+net: "examples/cifar10/cifar10_full_sigmoid_train_test.prototxt"
+# test_iter specifies how many forward passes the test should carry out.
+# In the case of CIFAR10, we have test batch size 100 and 100 test iterations,
+# covering the full 10,000 testing images.
+test_iter: 10
+# Carry out testing every 1000 training iterations.
+test_interval: 1000
+# The base learning rate, momentum and the weight decay of the network.
+base_lr: 0.001
+momentum: 0.9
+#weight_decay: 0.004
+# The learning rate policy
+lr_policy: "step"
+gamma: 1
+stepsize: 5000
+# Display every 200 iterations
+display: 100
+# The maximum number of iterations
+max_iter: 60000
+# snapshot intermediate results
+snapshot: 10000
+snapshot_prefix: "examples/cifar10_full_sigmoid"
+# solver mode: CPU or GPU
+solver_mode: GPU
diff --git a/examples/cifar10/cifar10_full_sigmoid_solver_bn.prototxt b/examples/cifar10/cifar10_full_sigmoid_solver_bn.prototxt
new file mode 100644
index 0000000..a57b280
--- /dev/null
+++ b/examples/cifar10/cifar10_full_sigmoid_solver_bn.prototxt
@@ -0,0 +1,28 @@
+# reduce learning rate after 120 epochs (60000 iters) by factor 0f 10
+# then another factor of 10 after 10 more epochs (5000 iters)
+
+# The train/test net protocol buffer definition
+net: "examples/cifar10/cifar10_full_sigmoid_train_test_bn.prototxt"
+# test_iter specifies how many forward passes the test should carry out.
+# In the case of CIFAR10, we have test batch size 100 and 100 test iterations,
+# covering the full 10,000 testing images.
+test_iter: 10
+# Carry out testing every 1000 training iterations.
+test_interval: 1000
+# The base learning rate, momentum and the weight decay of the network.
+base_lr: 0.001
+momentum: 0.9
+#weight_decay: 0.004
+# The learning rate policy
+lr_policy: "step"
+gamma: 1
+stepsize: 5000
+# Display every 200 iterations
+display: 100
+# The maximum number of iterations
+max_iter: 60000
+# snapshot intermediate results
+snapshot: 10000
+snapshot_prefix: "examples/cifar10_full_sigmoid_bn"
+# solver mode: CPU or GPU
+solver_mode: GPU
diff --git a/examples/cifar10/cifar10_full_sigmoid_train_test.prototxt b/examples/cifar10/cifar10_full_sigmoid_train_test.prototxt
new file mode 100644
index 0000000..fba69b8
--- /dev/null
+++ b/examples/cifar10/cifar10_full_sigmoid_train_test.prototxt
@@ -0,0 +1,212 @@
+name: "CIFAR10_full"
+layer {
+ name: "cifar"
+ type: "Data"
+ top: "data"
+ top: "label"
+ include {
+ phase: TRAIN
+ }
+ transform_param {
+ mean_file: "examples/cifar10/mean.binaryproto"
+ }
+ data_param {
+ source: "examples/cifar10/cifar10_train_lmdb"
+ batch_size: 111
+ backend: LMDB
+ }
+}
+layer {
+ name: "cifar"
+ type: "Data"
+ top: "data"
+ top: "label"
+ include {
+ phase: TEST
+ }
+ transform_param {
+ mean_file: "examples/cifar10/mean.binaryproto"
+ }
+ data_param {
+ source: "examples/cifar10/cifar10_test_lmdb"
+ batch_size: 1000
+ backend: LMDB
+ }
+}
+layer {
+ name: "conv1"
+ type: "Convolution"
+ bottom: "data"
+ top: "conv1"
+ param {
+ lr_mult: 1
+ }
+ param {
+ lr_mult: 2
+ }
+ convolution_param {
+ num_output: 32
+ pad: 2
+ kernel_size: 5
+ stride: 1
+ weight_filler {
+ type: "gaussian"
+ std: 0.0001
+ }
+ bias_filler {
+ type: "constant"
+ }
+ }
+}
+layer {
+ name: "pool1"
+ type: "Pooling"
+ bottom: "conv1"
+ top: "pool1"
+ pooling_param {
+ pool: MAX
+ kernel_size: 3
+ stride: 2
+ }
+}
+
+
+
+layer {
+ name: "Sigmoid1"
+ type: "Sigmoid"
+ bottom: "pool1"
+ top: "Sigmoid1"
+}
+
+layer {
+ name: "conv2"
+ type: "Convolution"
+ bottom: "Sigmoid1"
+ top: "conv2"
+ param {
+ lr_mult: 1
+ }
+ param {
+ lr_mult: 2
+ }
+ convolution_param {
+ num_output: 32
+ pad: 2
+ kernel_size: 5
+ stride: 1
+ weight_filler {
+ type: "gaussian"
+ std: 0.01
+ }
+ bias_filler {
+ type: "constant"
+ }
+ }
+}
+
+
+layer {
+ name: "Sigmoid2"
+ type: "Sigmoid"
+ bottom: "conv2"
+ top: "Sigmoid2"
+}
+layer {
+ name: "pool2"
+ type: "Pooling"
+ bottom: "Sigmoid2"
+ top: "pool2"
+ pooling_param {
+ pool: AVE
+ kernel_size: 3
+ stride: 2
+ }
+}
+layer {
+ name: "conv3"
+ type: "Convolution"
+ bottom: "pool2"
+ top: "conv3"
+ convolution_param {
+ num_output: 64
+ pad: 2
+ kernel_size: 5
+ stride: 1
+ weight_filler {
+ type: "gaussian"
+ std: 0.01
+ }
+ bias_filler {
+ type: "constant"
+ }
+ }
+ param {
+ lr_mult: 1
+ }
+ param {
+ lr_mult: 1
+ }
+
+}
+
+layer {
+ name: "Sigmoid3"
+ type: "Sigmoid"
+ bottom: "conv3"
+ top: "Sigmoid3"
+}
+
+layer {
+ name: "pool3"
+ type: "Pooling"
+ bottom: "Sigmoid3"
+ top: "pool3"
+ pooling_param {
+ pool: AVE
+ kernel_size: 3
+ stride: 2
+ }
+}
+
+layer {
+ name: "ip1"
+ type: "InnerProduct"
+ bottom: "pool3"
+ top: "ip1"
+ param {
+ lr_mult: 1
+ decay_mult: 0
+ }
+ param {
+ lr_mult: 2
+ decay_mult: 0
+ }
+ inner_product_param {
+ num_output: 10
+ weight_filler {
+ type: "gaussian"
+ std: 0.01
+ }
+ bias_filler {
+ type: "constant"
+ }
+ }
+}
+layer {
+ name: "accuracy"
+ type: "Accuracy"
+ bottom: "ip1"
+ bottom: "label"
+ top: "accuracy"
+ include {
+ phase: TEST
+ }
+}
+layer {
+ name: "loss"
+ type: "SoftmaxWithLoss"
+ bottom: "ip1"
+ bottom: "label"
+ top: "loss"
+}
diff --git a/examples/cifar10/cifar10_full_sigmoid_train_test_bn.prototxt b/examples/cifar10/cifar10_full_sigmoid_train_test_bn.prototxt
new file mode 100644
index 0000000..1a81075
--- /dev/null
+++ b/examples/cifar10/cifar10_full_sigmoid_train_test_bn.prototxt
@@ -0,0 +1,240 @@
+name: "CIFAR10_full"
+layer {
+ name: "cifar"
+ type: "Data"
+ top: "data"
+ top: "label"
+ include {
+ phase: TRAIN
+ }
+ transform_param {
+ mean_file: "examples/cifar10/mean.binaryproto"
+ }
+ data_param {
+ source: "examples/cifar10/cifar10_train_lmdb"
+ batch_size: 100
+ backend: LMDB
+ }
+}
+layer {
+ name: "cifar"
+ type: "Data"
+ top: "data"
+ top: "label"
+ include {
+ phase: TEST
+ }
+ transform_param {
+ mean_file: "examples/cifar10/mean.binaryproto"
+ }
+ data_param {
+ source: "examples/cifar10/cifar10_test_lmdb"
+ batch_size: 1000
+ backend: LMDB
+ }
+}
+layer {
+ name: "conv1"
+ type: "Convolution"
+ bottom: "data"
+ top: "conv1"
+ param {
+ lr_mult: 1
+ }
+ convolution_param {
+ num_output: 32
+ pad: 2
+ kernel_size: 5
+ stride: 1
+ bias_term: false
+ weight_filler {
+ type: "gaussian"
+ std: 0.0001
+ }
+ }
+}
+layer {
+ name: "pool1"
+ type: "Pooling"
+ bottom: "conv1"
+ top: "pool1"
+ pooling_param {
+ pool: MAX
+ kernel_size: 3
+ stride: 2
+ }
+}
+
+layer {
+ name: "bn1"
+ type: "BatchNorm"
+ bottom: "pool1"
+ top: "bn1"
+ param {
+ lr_mult: 0
+ }
+ param {
+ lr_mult: 0
+ }
+ param {
+ lr_mult: 0
+ }
+}
+
+layer {
+ name: "Sigmoid1"
+ type: "Sigmoid"
+ bottom: "bn1"
+ top: "Sigmoid1"
+}
+
+layer {
+ name: "conv2"
+ type: "Convolution"
+ bottom: "Sigmoid1"
+ top: "conv2"
+ param {
+ lr_mult: 1
+ }
+ convolution_param {
+ num_output: 32
+ pad: 2
+ kernel_size: 5
+ stride: 1
+ bias_term: false
+ weight_filler {
+ type: "gaussian"
+ std: 0.01
+ }
+ }
+}
+
+layer {
+ name: "bn2"
+ type: "BatchNorm"
+ bottom: "conv2"
+ top: "bn2"
+ param {
+ lr_mult: 0
+ }
+ param {
+ lr_mult: 0
+ }
+ param {
+ lr_mult: 0
+ }
+}
+
+layer {
+ name: "Sigmoid2"
+ type: "Sigmoid"
+ bottom: "bn2"
+ top: "Sigmoid2"
+}
+layer {
+ name: "pool2"
+ type: "Pooling"
+ bottom: "Sigmoid2"
+ top: "pool2"
+ pooling_param {
+ pool: AVE
+ kernel_size: 3
+ stride: 2
+ }
+}
+layer {
+ name: "conv3"
+ type: "Convolution"
+ bottom: "pool2"
+ top: "conv3"
+ param {
+ lr_mult: 1
+ }
+ convolution_param {
+ num_output: 64
+ pad: 2
+ kernel_size: 5
+ stride: 1
+ bias_term: false
+ weight_filler {
+ type: "gaussian"
+ std: 0.01
+ }
+ }
+}
+
+layer {
+ name: "bn3"
+ type: "BatchNorm"
+ bottom: "conv3"
+ top: "bn3"
+ param {
+ lr_mult: 0
+ }
+ param {
+ lr_mult: 0
+ }
+ param {
+ lr_mult: 0
+ }
+}
+
+layer {
+ name: "Sigmoid3"
+ type: "Sigmoid"
+ bottom: "bn3"
+ top: "Sigmoid3"
+}
+layer {
+ name: "pool3"
+ type: "Pooling"
+ bottom: "Sigmoid3"
+ top: "pool3"
+ pooling_param {
+ pool: AVE
+ kernel_size: 3
+ stride: 2
+ }
+}
+
+layer {
+ name: "ip1"
+ type: "InnerProduct"
+ bottom: "pool3"
+ top: "ip1"
+ param {
+ lr_mult: 1
+ decay_mult: 1
+ }
+ param {
+ lr_mult: 1
+ decay_mult: 0
+ }
+ inner_product_param {
+ num_output: 10
+ weight_filler {
+ type: "gaussian"
+ std: 0.01
+ }
+ bias_filler {
+ type: "constant"
+ }
+ }
+}
+layer {
+ name: "accuracy"
+ type: "Accuracy"
+ bottom: "ip1"
+ bottom: "label"
+ top: "accuracy"
+ include {
+ phase: TEST
+ }
+}
+layer {
+ name: "loss"
+ type: "SoftmaxWithLoss"
+ bottom: "ip1"
+ bottom: "label"
+ top: "loss"
+}
diff --git a/examples/cifar10/convert_cifar_data.cpp b/examples/cifar10/convert_cifar_data.cpp
index f4c42e4..e1b89f4 100644
--- a/examples/cifar10/convert_cifar_data.cpp
+++ b/examples/cifar10/convert_cifar_data.cpp
@@ -16,6 +16,7 @@
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/db.hpp"
+#include "caffe/util/format.hpp"
using caffe::Datum;
using boost::scoped_ptr;
@@ -52,19 +53,18 @@ void convert_dataset(const string& input_folder, const string& output_folder,
for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {
// Open files
LOG(INFO) << "Training Batch " << fileid + 1;
- snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
- std::ifstream data_file((input_folder + str_buffer).c_str(),
+ string batchFileName = input_folder + "/data_batch_"
+ + caffe::format_int(fileid+1) + ".bin";
+ std::ifstream data_file(batchFileName.c_str(),
std::ios::in | std::ios::binary);
CHECK(data_file) << "Unable to open train file #" << fileid + 1;
for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
read_image(&data_file, &label, str_buffer);
datum.set_label(label);
datum.set_data(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d",
- fileid * kCIFARBatchSize + itemid);
string out;
CHECK(datum.SerializeToString(&out));
- txn->Put(string(str_buffer, length), out);
+ txn->Put(caffe::format_int(fileid * kCIFARBatchSize + itemid, 5), out);
}
}
txn->Commit();
@@ -82,10 +82,9 @@ void convert_dataset(const string& input_folder, const string& output_folder,
read_image(&data_file, &label, str_buffer);
datum.set_label(label);
datum.set_data(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d", itemid);
string out;
CHECK(datum.SerializeToString(&out));
- txn->Put(string(str_buffer, length), out);
+ txn->Put(caffe::format_int(itemid, 5), out);
}
txn->Commit();
test_db->Close();
diff --git a/examples/cifar10/train_full_sigmoid.sh b/examples/cifar10/train_full_sigmoid.sh
new file mode 100755
index 0000000..9cff06d
--- /dev/null
+++ b/examples/cifar10/train_full_sigmoid.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+TOOLS=./build/tools
+
+$TOOLS/caffe train \
+ --solver=examples/cifar10/cifar10_full_sigmoid_solver.prototxt
+
diff --git a/examples/cifar10/train_full_sigmoid_bn.sh b/examples/cifar10/train_full_sigmoid_bn.sh
new file mode 100755
index 0000000..011387c
--- /dev/null
+++ b/examples/cifar10/train_full_sigmoid_bn.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+TOOLS=./build/tools
+
+$TOOLS/caffe train \
+ --solver=examples/cifar10/cifar10_full_sigmoid_solver_bn.prototxt
+
diff --git a/examples/cpp_classification/classification.cpp b/examples/cpp_classification/classification.cpp
index de48fb6..974662e 100644
--- a/examples/cpp_classification/classification.cpp
+++ b/examples/cpp_classification/classification.cpp
@@ -191,13 +191,13 @@ void Classifier::Preprocess(const cv::Mat& img,
/* Convert the input image to the input image format of the network. */
cv::Mat sample;
if (img.channels() == 3 && num_channels_ == 1)
- cv::cvtColor(img, sample, CV_BGR2GRAY);
+ cv::cvtColor(img, sample, cv::COLOR_BGR2GRAY);
else if (img.channels() == 4 && num_channels_ == 1)
- cv::cvtColor(img, sample, CV_BGRA2GRAY);
+ cv::cvtColor(img, sample, cv::COLOR_BGRA2GRAY);
else if (img.channels() == 4 && num_channels_ == 3)
- cv::cvtColor(img, sample, CV_BGRA2BGR);
+ cv::cvtColor(img, sample, cv::COLOR_BGRA2BGR);
else if (img.channels() == 1 && num_channels_ == 3)
- cv::cvtColor(img, sample, CV_GRAY2BGR);
+ cv::cvtColor(img, sample, cv::COLOR_GRAY2BGR);
else
sample = img;
diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp
index 8f29baf..16d2809 100644
--- a/examples/mnist/convert_mnist_data.cpp
+++ b/examples/mnist/convert_mnist_data.cpp
@@ -23,6 +23,7 @@
#include <string>
#include "caffe/proto/caffe.pb.h"
+#include "caffe/util/format.hpp"
#if defined(USE_LEVELDB) && defined(USE_LMDB)
@@ -108,8 +109,6 @@ void convert_dataset(const char* image_filename, const char* label_filename,
char label;
char* pixels = new char[rows * cols];
int count = 0;
- const int kMaxKeyLength = 10;
- char key_cstr[kMaxKeyLength];
string value;
Datum datum;
@@ -123,18 +122,17 @@ void convert_dataset(const char* image_filename, const char* label_filename,
label_file.read(&label, 1);
datum.set_data(pixels, rows*cols);
datum.set_label(label);
- snprintf(key_cstr, kMaxKeyLength, "%08d", item_id);
+ string key_str = caffe::format_int(item_id, 8);
datum.SerializeToString(&value);
- string keystr(key_cstr);
// Put in db
if (db_backend == "leveldb") { // leveldb
- batch->Put(keystr, value);
+ batch->Put(key_str, value);
} else if (db_backend == "lmdb") { // lmdb
mdb_data.mv_size = value.size();
mdb_data.mv_data = reinterpret_cast<void*>(&value[0]);
- mdb_key.mv_size = keystr.size();
- mdb_key.mv_data = reinterpret_cast<void*>(&keystr[0]);
+ mdb_key.mv_size = key_str.size();
+ mdb_key.mv_data = reinterpret_cast<void*>(&key_str[0]);
CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS)
<< "mdb_put failed";
} else {
diff --git a/examples/mnist/lenet_adadelta_solver.prototxt b/examples/mnist/lenet_adadelta_solver.prototxt
index 776d1e0..16176c0 100644
--- a/examples/mnist/lenet_adadelta_solver.prototxt
+++ b/examples/mnist/lenet_adadelta_solver.prototxt
@@ -20,5 +20,5 @@ snapshot: 5000
snapshot_prefix: "examples/mnist/lenet_adadelta"
# solver mode: CPU or GPU
solver_mode: GPU
-solver_type: ADADELTA
+type: "AdaDelta"
delta: 1e-6
diff --git a/examples/mnist/lenet_solver_adam.prototxt b/examples/mnist/lenet_solver_adam.prototxt
index d22c571..4b5336b 100644
--- a/examples/mnist/lenet_solver_adam.prototxt
+++ b/examples/mnist/lenet_solver_adam.prototxt
@@ -22,5 +22,5 @@ max_iter: 10000
snapshot: 5000
snapshot_prefix: "examples/mnist/lenet"
# solver mode: CPU or GPU
-solver_type: ADAM
+type: "Adam"
solver_mode: GPU
diff --git a/examples/mnist/lenet_solver_rmsprop.prototxt b/examples/mnist/lenet_solver_rmsprop.prototxt
index 74dadc5..924b72d 100644
--- a/examples/mnist/lenet_solver_rmsprop.prototxt
+++ b/examples/mnist/lenet_solver_rmsprop.prototxt
@@ -23,5 +23,5 @@ snapshot: 5000
snapshot_prefix: "examples/mnist/lenet_rmsprop"
# solver mode: CPU or GPU
solver_mode: GPU
-solver_type: RMSPROP
+type: "RMSProp"
rms_decay: 0.98
diff --git a/examples/mnist/lenet_stepearly_solver.prototxt b/examples/mnist/lenet_stepearly_solver.prototxt
deleted file mode 100644
index efc6a33..0000000
--- a/examples/mnist/lenet_stepearly_solver.prototxt
+++ /dev/null
@@ -1,28 +0,0 @@
-# The training protocol buffer definition
-train_net: "lenet_train.prototxt"
-# The testing protocol buffer definition
-test_net: "lenet_test.prototxt"
-# test_iter specifies how many forward passes the test should carry out.
-# In the case of MNIST, we have test batch size 100 and 100 test iterations,
-# covering the full 10,000 testing images.
-test_iter: 100
-# Carry out testing every 500 training iterations.
-test_interval: 500
-# The base learning rate, momentum and the weight decay of the network.
-base_lr: 0.01
-momentum: 0.9
-weight_decay: 0.0005
-# The learning rate policy
-lr_policy: "stepearly"
-gamma: 0.9
-stepearly: 1
-# Display every 100 iterations
-display: 100
-# The maximum number of iterations
-max_iter: 10000
-# snapshot intermediate results
-snapshot: 5000
-snapshot_prefix: "lenet"
-# solver mode: 0 for CPU and 1 for GPU
-solver_mode: 1
-device_id: 1
diff --git a/examples/mnist/mnist_autoencoder_solver_adadelta.prototxt b/examples/mnist/mnist_autoencoder_solver_adadelta.prototxt
index 065647d..26c4084 100644
--- a/examples/mnist/mnist_autoencoder_solver_adadelta.prototxt
+++ b/examples/mnist/mnist_autoencoder_solver_adadelta.prototxt
@@ -16,4 +16,4 @@ snapshot: 10000
snapshot_prefix: "examples/mnist/mnist_autoencoder_adadelta_train"
# solver mode: CPU or GPU
solver_mode: GPU
-solver_type: ADADELTA
+type: "AdaDelta"
diff --git a/examples/mnist/mnist_autoencoder_solver_adagrad.prototxt b/examples/mnist/mnist_autoencoder_solver_adagrad.prototxt
index cc0ed9e..065cdb2 100644
--- a/examples/mnist/mnist_autoencoder_solver_adagrad.prototxt
+++ b/examples/mnist/mnist_autoencoder_solver_adagrad.prototxt
@@ -14,4 +14,4 @@ snapshot: 10000
snapshot_prefix: "examples/mnist/mnist_autoencoder_adagrad_train"
# solver mode: CPU or GPU
solver_mode: GPU
-solver_type: ADAGRAD
+type: "AdaGrad"
diff --git a/examples/mnist/mnist_autoencoder_solver_nesterov.prototxt b/examples/mnist/mnist_autoencoder_solver_nesterov.prototxt
index 2a59fd4..c95e3fe 100644
--- a/examples/mnist/mnist_autoencoder_solver_nesterov.prototxt
+++ b/examples/mnist/mnist_autoencoder_solver_nesterov.prototxt
@@ -17,4 +17,4 @@ snapshot_prefix: "examples/mnist/mnist_autoencoder_nesterov_train"
momentum: 0.95
# solver mode: CPU or GPU
solver_mode: GPU
-solver_type: NESTEROV
+type: "Nesterov"
diff --git a/examples/siamese/convert_mnist_siamese_data.cpp b/examples/siamese/convert_mnist_siamese_data.cpp
index ad08036..928b3fb 100644
--- a/examples/siamese/convert_mnist_siamese_data.cpp
+++ b/examples/siamese/convert_mnist_siamese_data.cpp
@@ -13,6 +13,7 @@
#include "stdint.h"
#include "caffe/proto/caffe.pb.h"
+#include "caffe/util/format.hpp"
#include "caffe/util/math_functions.hpp"
#ifdef USE_LEVELDB
@@ -75,8 +76,6 @@ void convert_dataset(const char* image_filename, const char* label_filename,
char label_i;
char label_j;
char* pixels = new char[2 * rows * cols];
- const int kMaxKeyLength = 10;
- char key[kMaxKeyLength];
std::string value;
caffe::Datum datum;
@@ -99,8 +98,8 @@ void convert_dataset(const char* image_filename, const char* label_filename,
datum.set_label(0);
}
datum.SerializeToString(&value);
- snprintf(key, kMaxKeyLength, "%08d", itemid);
- db->Put(leveldb::WriteOptions(), std::string(key), value);
+ std::string key_str = caffe::format_int(itemid, 8);
+ db->Put(leveldb::WriteOptions(), key_str, value);
}
delete db;
diff --git a/examples/web_demo/requirements.txt b/examples/web_demo/requirements.txt
index 8fb1d2c..43e1b98 100644
--- a/examples/web_demo/requirements.txt
+++ b/examples/web_demo/requirements.txt
@@ -4,3 +4,4 @@ tornado
numpy
pandas
pillow
+pyyaml
diff --git a/include/caffe/blob.hpp b/include/caffe/blob.hpp
index fea5117..af360ac 100644
--- a/include/caffe/blob.hpp
+++ b/include/caffe/blob.hpp
@@ -8,7 +8,6 @@
#include "caffe/common.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/syncedmem.hpp"
-#include "caffe/util/math_functions.hpp"
const int kMaxBlobAxes = 32;
diff --git a/include/caffe/caffe.hpp b/include/caffe/caffe.hpp
index 68a5e1d..0688209 100644
--- a/include/caffe/caffe.hpp
+++ b/include/caffe/caffe.hpp
@@ -13,8 +13,9 @@
#include "caffe/parallel.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/solver.hpp"
+#include "caffe/solver_factory.hpp"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/io.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/util/upgrade_proto.hpp"
#endif // CAFFE_CAFFE_HPP_
diff --git a/include/caffe/common_layers.hpp b/include/caffe/common_layers.hpp
deleted file mode 100644
index 89bab8d..0000000
--- a/include/caffe/common_layers.hpp
+++ /dev/null
@@ -1,678 +0,0 @@
-#ifndef CAFFE_COMMON_LAYERS_HPP_
-#define CAFFE_COMMON_LAYERS_HPP_
-
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/loss_layers.hpp"
-#include "caffe/neuron_layers.hpp"
-#include "caffe/proto/caffe.pb.h"
-
-namespace caffe {
-
-/**
- * @brief Compute the index of the @f$ K @f$ max values for each datum across
- * all dimensions @f$ (C \times H \times W) @f$.
- *
- * Intended for use after a classification layer to produce a prediction.
- * If parameter out_max_val is set to true, output is a vector of pairs
- * (max_ind, max_val) for each image.
- *
- * NOTE: does not implement Backwards operation.
- */
-template <typename Dtype>
-class ArgMaxLayer : public Layer<Dtype> {
- public:
- /**
- * @param param provides ArgMaxParameter argmax_param,
- * with ArgMaxLayer options:
- * - top_k (\b optional uint, default 1).
- * the number @f$ K @f$ of maximal items to output.
- * - out_max_val (\b optional bool, default false).
- * if set, output a vector of pairs (max_ind, max_val) for each image.
- */
- explicit ArgMaxLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "ArgMax"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times 1 \times K \times 1) @f$ or, if out_max_val
- * @f$ (N \times 2 \times K \times 1) @f$
- * the computed outputs @f$
- * y_n = \arg\max\limits_i x_{ni}
- * @f$ (for @f$ K = 1 @f$).
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- /// @brief Not implemented (non-differentiable function)
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
- NOT_IMPLEMENTED;
- }
- bool out_max_val_;
- size_t top_k_;
-};
-
-/**
- * @brief Takes at least two Blob%s and concatenates them along either the num
- * or channel dimension, outputting the result.
- */
-template <typename Dtype>
-class ConcatLayer : public Layer<Dtype> {
- public:
- explicit ConcatLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Concat"; }
- virtual inline int MinBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 2+)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x_1 @f$
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x_2 @f$
- * -# ...
- * - K @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x_K @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (KN \times C \times H \times W) @f$ if axis == 0, or
- * @f$ (N \times KC \times H \times W) @f$ if axis == 1:
- * the concatenated output @f$
- * y = [\begin{array}{cccc} x_1 & x_2 & ... & x_K \end{array}]
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the concatenate inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (KN \times C \times H \times W) @f$ if axis == 0, or
- * @f$ (N \times KC \times H \times W) @f$ if axis == 1:
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to concatenated outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length K), into which the top gradient
- * @f$ \frac{\partial E}{\partial y} @f$ is deconcatenated back to the
- * inputs @f$
- * \left[ \begin{array}{cccc}
- * \frac{\partial E}{\partial x_1} &
- * \frac{\partial E}{\partial x_2} &
- * ... &
- * \frac{\partial E}{\partial x_K}
- * \end{array} \right] =
- * \frac{\partial E}{\partial y}
- * @f$
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int count_;
- int num_concats_;
- int concat_input_size_;
- int concat_axis_;
-};
-
-/**
- * @brief Compute elementwise operations, such as product and sum,
- * along multiple input Blobs.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class EltwiseLayer : public Layer<Dtype> {
- public:
- explicit EltwiseLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Eltwise"; }
- virtual inline int MinBottomBlobs() const { return 2; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- EltwiseParameter_EltwiseOp op_;
- vector<Dtype> coeffs_;
- Blob<int> max_idx_;
-
- bool stable_prod_grad_;
-};
-
-/**
- * @brief A layer for learning "embeddings" of one-hot vector input.
- * Equivalent to an InnerProductLayer with one-hot vectors as input, but
- * for efficiency the input is the "hot" index of each column itself.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class EmbedLayer : public Layer<Dtype> {
- public:
- explicit EmbedLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Embed"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int M_;
- int K_;
- int N_;
- bool bias_term_;
- Blob<Dtype> bias_multiplier_;
-};
-
-/**
- * @brief Takes two+ Blobs, interprets last Blob as a selector and
- * filter remaining Blobs accordingly with selector data (0 means that
- * the corresponding item has to be filtered, non-zero means that corresponding
- * item needs to stay).
- */
-template <typename Dtype>
-class FilterLayer : public Layer<Dtype> {
- public:
- explicit FilterLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Filter"; }
- virtual inline int MinBottomBlobs() const { return 2; }
- virtual inline int MinTopBlobs() const { return 1; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 2+)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs to be filtered @f$ x_1 @f$
- * -# ...
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs to be filtered @f$ x_K @f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the selector blob
- * @param top output Blob vector (length 1+)
- * -# @f$ (S \times C \times H \times W) @f$ ()
- * the filtered output @f$ x_1 @f$
- * where S is the number of items
- * that haven't been filtered
- * @f$ (S \times C \times H \times W) @f$
- * the filtered output @f$ x_K @f$
- * where S is the number of items
- * that haven't been filtered
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the forwarded inputs.
- *
- * @param top output Blob vector (length 1+), providing the error gradient with
- * respect to the outputs
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 2+), into which the top error
- * gradient is copied
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool first_reshape_;
- vector<int> indices_to_forward_;
-};
-
-/**
- * @brief Reshapes the input Blob into flat vectors.
- *
- * Note: because this layer does not change the input values -- merely the
- * dimensions -- it can simply copy the input. The copy happens "virtually"
- * (thus taking effectively 0 real time) by setting, in Forward, the data
- * pointer of the top Blob to that of the bottom Blob (see Blob::ShareData),
- * and in Backward, the diff pointer of the bottom Blob to that of the top Blob
- * (see Blob::ShareDiff).
- */
-template <typename Dtype>
-class FlattenLayer : public Layer<Dtype> {
- public:
- explicit FlattenLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Flatten"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 2+)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times CHW \times 1 \times 1) @f$
- * the outputs -- i.e., the (virtually) copied, flattened inputs
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the concatenate inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length K), into which the top error
- * gradient is (virtually) copied
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-/**
- * @brief Also known as a "fully-connected" layer, computes an inner product
- * with a set of learned weights, and (optionally) adds biases.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class InnerProductLayer : public Layer<Dtype> {
- public:
- explicit InnerProductLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "InnerProduct"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int M_;
- int K_;
- int N_;
- bool bias_term_;
- Blob<Dtype> bias_multiplier_;
-};
-
-/**
- * @brief Normalizes the input to have 0-mean and/or unit (1) variance.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class MVNLayer : public Layer<Dtype> {
- public:
- explicit MVNLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "MVN"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- Blob<Dtype> mean_, variance_, temp_;
-
- /// sum_multiplier is used to carry out sum using BLAS
- Blob<Dtype> sum_multiplier_;
- Dtype eps_;
-};
-
-/*
- * @brief Reshapes the input Blob into an arbitrary-sized output Blob.
- *
- * Note: similarly to FlattenLayer, this layer does not change the input values
- * (see FlattenLayer, Blob::ShareData and Blob::ShareDiff).
- */
-template <typename Dtype>
-class ReshapeLayer : public Layer<Dtype> {
- public:
- explicit ReshapeLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Reshape"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
-
- /// @brief vector of axes indices whose dimensions we'll copy from the bottom
- vector<int> copy_axes_;
- /// @brief the index of the axis whose dimension we infer, or -1 if none
- int inferred_axis_;
- /// @brief the product of the "constant" output dimensions
- int constant_count_;
-};
-
-/**
- * @brief Compute "reductions" -- operations that return a scalar output Blob
- * for an input Blob of arbitrary size, such as the sum, absolute sum,
- * and sum of squares.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class ReductionLayer : public Layer<Dtype> {
- public:
- explicit ReductionLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Reduction"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- /// @brief the reduction operation performed by the layer
- ReductionParameter_ReductionOp op_;
- /// @brief a scalar coefficient applied to all outputs
- Dtype coeff_;
- /// @brief the index of the first input axis to reduce
- int axis_;
- /// @brief the number of reductions performed
- int num_;
- /// @brief the input size of each reduction
- int dim_;
- /// @brief a helper Blob used for summation (op_ == SUM)
- Blob<Dtype> sum_multiplier_;
-};
-
-/**
- * @brief Ignores bottom blobs while producing no top blobs. (This is useful
- * to suppress outputs during testing.)
- */
-template <typename Dtype>
-class SilenceLayer : public Layer<Dtype> {
- public:
- explicit SilenceLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
-
- virtual inline const char* type() const { return "Silence"; }
- virtual inline int MinBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 0; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
- // We can't define Forward_gpu here, since STUB_GPU will provide
- // its own definition for CPU_ONLY mode.
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-/**
- * @brief Computes the softmax function.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class SoftmaxLayer : public Layer<Dtype> {
- public:
- explicit SoftmaxLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Softmax"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int outer_num_;
- int inner_num_;
- int softmax_axis_;
- /// sum_multiplier is used to carry out sum using BLAS
- Blob<Dtype> sum_multiplier_;
- /// scale is an intermediate Blob to hold temporary results.
- Blob<Dtype> scale_;
-};
-
-#ifdef USE_CUDNN
-/**
- * @brief cuDNN implementation of SoftmaxLayer.
- * Fallback to SoftmaxLayer for CPU mode.
- */
-template <typename Dtype>
-class CuDNNSoftmaxLayer : public SoftmaxLayer<Dtype> {
- public:
- explicit CuDNNSoftmaxLayer(const LayerParameter& param)
- : SoftmaxLayer<Dtype>(param), handles_setup_(false) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual ~CuDNNSoftmaxLayer();
-
- protected:
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool handles_setup_;
- cudnnHandle_t handle_;
- cudnnTensorDescriptor_t bottom_desc_;
- cudnnTensorDescriptor_t top_desc_;
-};
-#endif
-
-/**
- * @brief Creates a "split" path in the network by copying the bottom Blob
- * into multiple top Blob%s to be used by multiple consuming layers.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class SplitLayer : public Layer<Dtype> {
- public:
- explicit SplitLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Split"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int MinTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int count_;
-};
-
-/**
- * @brief Takes a Blob and slices it along either the num or channel dimension,
- * outputting multiple sliced Blob results.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class SliceLayer : public Layer<Dtype> {
- public:
- explicit SliceLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Slice"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int MinTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int count_;
- int num_slices_;
- int slice_size_;
- int slice_axis_;
- vector<int> slice_point_;
-};
-
-/**
- * @brief Copy a Blob along specified dimensions.
- */
-template <typename Dtype>
-class TileLayer : public Layer<Dtype> {
- public:
- explicit TileLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Tile"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- unsigned int axis_, tiles_, outer_dim_, inner_dim_;
-};
-
-} // namespace caffe
-
-#endif // CAFFE_COMMON_LAYERS_HPP_
diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp
deleted file mode 100644
index 90fd0d1..0000000
--- a/include/caffe/data_layers.hpp
+++ /dev/null
@@ -1,344 +0,0 @@
-#ifndef CAFFE_DATA_LAYERS_HPP_
-#define CAFFE_DATA_LAYERS_HPP_
-
-#include <string>
-#include <utility>
-#include <vector>
-#include "hdf5.h"
-
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/data_reader.hpp"
-#include "caffe/data_transformer.hpp"
-#include "caffe/filler.hpp"
-#include "caffe/internal_thread.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/proto/caffe.pb.h"
-#include "caffe/util/blocking_queue.hpp"
-#include "caffe/util/db.hpp"
-
-namespace caffe {
-
-/**
- * @brief Provides base for data layers that feed blobs to the Net.
- *
- * TODO(dox): thorough documentation for Forward and proto params.
- */
-template <typename Dtype>
-class BaseDataLayer : public Layer<Dtype> {
- public:
- explicit BaseDataLayer(const LayerParameter& param);
- // LayerSetUp: implements common data layer setup functionality, and calls
- // DataLayerSetUp to do special data layer setup for individual layer types.
- // This method may not be overridden except by the BasePrefetchingDataLayer.
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- // Data layers should be shared by multiple solvers in parallel
- virtual inline bool ShareInParallel() const { return true; }
- virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
- // Data layers have no bottoms, so reshaping is trivial.
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
-
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
-
- protected:
- TransformationParameter transform_param_;
- shared_ptr<DataTransformer<Dtype> > data_transformer_;
- bool output_labels_;
-};
-
-template <typename Dtype>
-class Batch {
- public:
- Blob<Dtype> data_, label_;
-};
-
-template <typename Dtype>
-class BasePrefetchingDataLayer :
- public BaseDataLayer<Dtype>, public InternalThread {
- public:
- explicit BasePrefetchingDataLayer(const LayerParameter& param);
- // LayerSetUp: implements common data layer setup functionality, and calls
- // DataLayerSetUp to do special data layer setup for individual layer types.
- // This method may not be overridden.
- void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- // Prefetches batches (asynchronously if to GPU memory)
- static const int PREFETCH_COUNT = 3;
-
- protected:
- virtual void InternalThreadEntry();
- virtual void load_batch(Batch<Dtype>* batch) = 0;
-
- Batch<Dtype> prefetch_[PREFETCH_COUNT];
- BlockingQueue<Batch<Dtype>*> prefetch_free_;
- BlockingQueue<Batch<Dtype>*> prefetch_full_;
-
- Blob<Dtype> transformed_data_;
-};
-
-template <typename Dtype>
-class DataLayer : public BasePrefetchingDataLayer<Dtype> {
- public:
- explicit DataLayer(const LayerParameter& param);
- virtual ~DataLayer();
- virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- // DataLayer uses DataReader instead for sharing for parallelism
- virtual inline bool ShareInParallel() const { return false; }
- virtual inline const char* type() const { return "Data"; }
- virtual inline int ExactNumBottomBlobs() const { return 0; }
- virtual inline int MinTopBlobs() const { return 1; }
- virtual inline int MaxTopBlobs() const { return 2; }
-
- protected:
- virtual void load_batch(Batch<Dtype>* batch);
-
- DataReader reader_;
-};
-
-/**
- * @brief Provides data to the Net generated by a Filler.
- *
- * TODO(dox): thorough documentation for Forward and proto params.
- */
-template <typename Dtype>
-class DummyDataLayer : public Layer<Dtype> {
- public:
- explicit DummyDataLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- // Data layers should be shared by multiple solvers in parallel
- virtual inline bool ShareInParallel() const { return true; }
- // Data layers have no bottoms, so reshaping is trivial.
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
-
- virtual inline const char* type() const { return "DummyData"; }
- virtual inline int ExactNumBottomBlobs() const { return 0; }
- virtual inline int MinTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
-
- vector<shared_ptr<Filler<Dtype> > > fillers_;
- vector<bool> refill_;
-};
-
-/**
- * @brief Provides data to the Net from HDF5 files.
- *
- * TODO(dox): thorough documentation for Forward and proto params.
- */
-template <typename Dtype>
-class HDF5DataLayer : public Layer<Dtype> {
- public:
- explicit HDF5DataLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual ~HDF5DataLayer();
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- // Data layers should be shared by multiple solvers in parallel
- virtual inline bool ShareInParallel() const { return true; }
- // Data layers have no bottoms, so reshaping is trivial.
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
-
- virtual inline const char* type() const { return "HDF5Data"; }
- virtual inline int ExactNumBottomBlobs() const { return 0; }
- virtual inline int MinTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
- virtual void LoadHDF5FileData(const char* filename);
-
- std::vector<std::string> hdf_filenames_;
- unsigned int num_files_;
- unsigned int current_file_;
- hsize_t current_row_;
- std::vector<shared_ptr<Blob<Dtype> > > hdf_blobs_;
- std::vector<unsigned int> data_permutation_;
- std::vector<unsigned int> file_permutation_;
-};
-
-/**
- * @brief Write blobs to disk as HDF5 files.
- *
- * TODO(dox): thorough documentation for Forward and proto params.
- */
-template <typename Dtype>
-class HDF5OutputLayer : public Layer<Dtype> {
- public:
- explicit HDF5OutputLayer(const LayerParameter& param)
- : Layer<Dtype>(param), file_opened_(false) {}
- virtual ~HDF5OutputLayer();
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- // Data layers should be shared by multiple solvers in parallel
- virtual inline bool ShareInParallel() const { return true; }
- // Data layers have no bottoms, so reshaping is trivial.
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top) {}
-
- virtual inline const char* type() const { return "HDF5Output"; }
- // TODO: no limit on the number of blobs
- virtual inline int ExactNumBottomBlobs() const { return 2; }
- virtual inline int ExactNumTopBlobs() const { return 0; }
-
- inline std::string file_name() const { return file_name_; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void SaveBlobs();
-
- bool file_opened_;
- std::string file_name_;
- hid_t file_id_;
- Blob<Dtype> data_blob_;
- Blob<Dtype> label_blob_;
-};
-
-/**
- * @brief Provides data to the Net from image files.
- *
- * TODO(dox): thorough documentation for Forward and proto params.
- */
-template <typename Dtype>
-class ImageDataLayer : public BasePrefetchingDataLayer<Dtype> {
- public:
- explicit ImageDataLayer(const LayerParameter& param)
- : BasePrefetchingDataLayer<Dtype>(param) {}
- virtual ~ImageDataLayer();
- virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "ImageData"; }
- virtual inline int ExactNumBottomBlobs() const { return 0; }
- virtual inline int ExactNumTopBlobs() const { return 2; }
-
- protected:
- shared_ptr<Caffe::RNG> prefetch_rng_;
- virtual void ShuffleImages();
- virtual void load_batch(Batch<Dtype>* batch);
-
- vector<std::pair<std::string, int> > lines_;
- int lines_id_;
-};
-
-/**
- * @brief Provides data to the Net from memory.
- *
- * TODO(dox): thorough documentation for Forward and proto params.
- */
-template <typename Dtype>
-class MemoryDataLayer : public BaseDataLayer<Dtype> {
- public:
- explicit MemoryDataLayer(const LayerParameter& param)
- : BaseDataLayer<Dtype>(param), has_new_data_(false) {}
- virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "MemoryData"; }
- virtual inline int ExactNumBottomBlobs() const { return 0; }
- virtual inline int ExactNumTopBlobs() const { return 2; }
-
- virtual void AddDatumVector(const vector<Datum>& datum_vector);
-#ifdef USE_OPENCV
- virtual void AddMatVector(const vector<cv::Mat>& mat_vector,
- const vector<int>& labels);
-#endif // USE_OPENCV
-
- // Reset should accept const pointers, but can't, because the memory
- // will be given to Blob, which is mutable
- void Reset(Dtype* data, Dtype* label, int n);
- void set_batch_size(int new_size);
-
- int batch_size() { return batch_size_; }
- int channels() { return channels_; }
- int height() { return height_; }
- int width() { return width_; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- int batch_size_, channels_, height_, width_, size_;
- Dtype* data_;
- Dtype* labels_;
- int n_;
- size_t pos_;
- Blob<Dtype> added_data_;
- Blob<Dtype> added_label_;
- bool has_new_data_;
-};
-
-/**
- * @brief Provides data to the Net from windows of images files, specified
- * by a window data file.
- *
- * TODO(dox): thorough documentation for Forward and proto params.
- */
-template <typename Dtype>
-class WindowDataLayer : public BasePrefetchingDataLayer<Dtype> {
- public:
- explicit WindowDataLayer(const LayerParameter& param)
- : BasePrefetchingDataLayer<Dtype>(param) {}
- virtual ~WindowDataLayer();
- virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "WindowData"; }
- virtual inline int ExactNumBottomBlobs() const { return 0; }
- virtual inline int ExactNumTopBlobs() const { return 2; }
-
- protected:
- virtual unsigned int PrefetchRand();
- virtual void load_batch(Batch<Dtype>* batch);
-
- shared_ptr<Caffe::RNG> prefetch_rng_;
- vector<std::pair<std::string, vector<int> > > image_database_;
- enum WindowField { IMAGE_INDEX, LABEL, OVERLAP, X1, Y1, X2, Y2, NUM };
- vector<vector<float> > fg_windows_;
- vector<vector<float> > bg_windows_;
- Blob<Dtype> data_mean_;
- vector<Dtype> mean_values_;
- bool has_mean_file_;
- bool has_mean_values_;
- bool cache_images_;
- vector<std::pair<std::string, Datum > > image_database_cache_;
-};
-
-} // namespace caffe
-
-#endif // CAFFE_DATA_LAYERS_HPP_
diff --git a/include/caffe/filler.hpp b/include/caffe/filler.hpp
index 888f4a4..dad9ad4 100644
--- a/include/caffe/filler.hpp
+++ b/include/caffe/filler.hpp
@@ -8,7 +8,6 @@
#include <string>
#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/syncedmem.hpp"
#include "caffe/util/math_functions.hpp"
diff --git a/include/caffe/layer.hpp b/include/caffe/layer.hpp
index a0d1d4e..10f353f 100644
--- a/include/caffe/layer.hpp
+++ b/include/caffe/layer.hpp
@@ -9,7 +9,7 @@
#include "caffe/common.hpp"
#include "caffe/layer_factory.hpp"
#include "caffe/proto/caffe.pb.h"
-#include "caffe/util/device_alternate.hpp"
+#include "caffe/util/math_functions.hpp"
/**
Forward declare boost::thread instead of including boost/thread.hpp
diff --git a/include/caffe/layer_factory.hpp b/include/caffe/layer_factory.hpp
index 2c2fde4..f385afc 100644
--- a/include/caffe/layer_factory.hpp
+++ b/include/caffe/layer_factory.hpp
@@ -44,6 +44,7 @@
#include <vector>
#include "caffe/common.hpp"
+#include "caffe/layer.hpp"
#include "caffe/proto/caffe.pb.h"
namespace caffe {
diff --git a/include/caffe/layers/absval_layer.hpp b/include/caffe/layers/absval_layer.hpp
new file mode 100644
index 0000000..9b5305d
--- /dev/null
+++ b/include/caffe/layers/absval_layer.hpp
@@ -0,0 +1,68 @@
+#ifndef CAFFE_ABSVAL_LAYER_HPP_
+#define CAFFE_ABSVAL_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes @f$ y = |x| @f$
+ *
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$ y = |x| @f$
+ */
+template <typename Dtype>
+class AbsValLayer : public NeuronLayer<Dtype> {
+ public:
+ explicit AbsValLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "AbsVal"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ /// @copydoc AbsValLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the absolute value inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x} =
+ * \mathrm{sign}(x) \frac{\partial E}{\partial y}
+ * @f$ if propagate_down[0]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_ABSVAL_LAYER_HPP_
diff --git a/include/caffe/layers/accuracy_layer.hpp b/include/caffe/layers/accuracy_layer.hpp
new file mode 100644
index 0000000..fe2adb9
--- /dev/null
+++ b/include/caffe/layers/accuracy_layer.hpp
@@ -0,0 +1,95 @@
+#ifndef CAFFE_ACCURACY_LAYER_HPP_
+#define CAFFE_ACCURACY_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes the classification accuracy for a one-of-many
+ * classification task.
+ */
+template <typename Dtype>
+class AccuracyLayer : public Layer<Dtype> {
+ public:
+ /**
+ * @param param provides AccuracyParameter accuracy_param,
+ * with AccuracyLayer options:
+ * - top_k (\b optional, default 1).
+ * Sets the maximum rank @f$ k @f$ at which a prediction is considered
+ * correct. For example, if @f$ k = 5 @f$, a prediction is counted
+ * correct if the correct label is among the top 5 predicted labels.
+ */
+ explicit AccuracyLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Accuracy"; }
+ virtual inline int ExactNumBottomBlobs() const { return 2; }
+
+ // If there are two top blobs, then the second blob will contain
+ // accuracies per class.
+ virtual inline int MinTopBlobs() const { return 1; }
+ virtual inline int MaxTopBlos() const { return 2; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ x @f$, a Blob with values in
+ * @f$ [-\infty, +\infty] @f$ indicating the predicted score for each of
+ * the @f$ K = CHW @f$ classes. Each @f$ x_n @f$ is mapped to a predicted
+ * label @f$ \hat{l}_n @f$ given by its maximal index:
+ * @f$ \hat{l}_n = \arg\max\limits_k x_{nk} @f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels @f$ l @f$, an integer-valued Blob with values
+ * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
+ * indicating the correct class label among the @f$ K @f$ classes
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed accuracy: @f$
+ * \frac{1}{N} \sum\limits_{n=1}^N \delta\{ \hat{l}_n = l_n \}
+ * @f$, where @f$
+ * \delta\{\mathrm{condition}\} = \left\{
+ * \begin{array}{lr}
+ * 1 & \mbox{if condition} \\
+ * 0 & \mbox{otherwise}
+ * \end{array} \right.
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+
+ /// @brief Not implemented -- AccuracyLayer cannot be used as a loss.
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
+ for (int i = 0; i < propagate_down.size(); ++i) {
+ if (propagate_down[i]) { NOT_IMPLEMENTED; }
+ }
+ }
+
+ int label_axis_, outer_num_, inner_num_;
+
+ int top_k_;
+
+ /// Whether to ignore instances with a certain label.
+ bool has_ignore_label_;
+ /// The label indicating that an instance should be ignored.
+ int ignore_label_;
+ /// Keeps counts of the number of samples per class.
+ Blob<Dtype> nums_buffer_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_ACCURACY_LAYER_HPP_
diff --git a/include/caffe/layers/argmax_layer.hpp b/include/caffe/layers/argmax_layer.hpp
new file mode 100644
index 0000000..4fef363
--- /dev/null
+++ b/include/caffe/layers/argmax_layer.hpp
@@ -0,0 +1,77 @@
+#ifndef CAFFE_ARGMAX_LAYER_HPP_
+#define CAFFE_ARGMAX_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Compute the index of the @f$ K @f$ max values for each datum across
+ * all dimensions @f$ (C \times H \times W) @f$.
+ *
+ * Intended for use after a classification layer to produce a prediction.
+ * If parameter out_max_val is set to true, output is a vector of pairs
+ * (max_ind, max_val) for each image. The axis parameter specifies an axis
+ * along which to maximise.
+ *
+ * NOTE: does not implement Backwards operation.
+ */
+template <typename Dtype>
+class ArgMaxLayer : public Layer<Dtype> {
+ public:
+ /**
+ * @param param provides ArgMaxParameter argmax_param,
+ * with ArgMaxLayer options:
+ * - top_k (\b optional uint, default 1).
+ * the number @f$ K @f$ of maximal items to output.
+ * - out_max_val (\b optional bool, default false).
+ * if set, output a vector of pairs (max_ind, max_val) unless axis is set then
+ * output max_val along the specified axis.
+ * - axis (\b optional int).
+ * if set, maximise along the specified axis else maximise the flattened
+ * trailing dimensions for each index of the first / num dimension.
+ */
+ explicit ArgMaxLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "ArgMax"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times 1 \times K) @f$ or, if out_max_val
+ * @f$ (N \times 2 \times K) @f$ unless axis set than e.g.
+ * @f$ (N \times K \times H \times W) @f$ if axis == 1
+ * the computed outputs @f$
+ * y_n = \arg\max\limits_i x_{ni}
+ * @f$ (for @f$ K = 1 @f$).
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ /// @brief Not implemented (non-differentiable function)
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
+ NOT_IMPLEMENTED;
+ }
+ bool out_max_val_;
+ size_t top_k_;
+ bool has_axis_;
+ int axis_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_ARGMAX_LAYER_HPP_
diff --git a/include/caffe/layers/base_conv_layer.hpp b/include/caffe/layers/base_conv_layer.hpp
new file mode 100644
index 0000000..f3def16
--- /dev/null
+++ b/include/caffe/layers/base_conv_layer.hpp
@@ -0,0 +1,168 @@
+#ifndef CAFFE_BASE_CONVOLUTION_LAYER_HPP_
+#define CAFFE_BASE_CONVOLUTION_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+#include "caffe/util/im2col.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Abstract base class that factors out the BLAS code common to
+ * ConvolutionLayer and DeconvolutionLayer.
+ */
+template <typename Dtype>
+class BaseConvolutionLayer : public Layer<Dtype> {
+ public:
+ explicit BaseConvolutionLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline int MinBottomBlobs() const { return 1; }
+ virtual inline int MinTopBlobs() const { return 1; }
+ virtual inline bool EqualNumBottomTopBlobs() const { return true; }
+
+ protected:
+ // Helper functions that abstract away the column buffer and gemm arguments.
+ // The last argument in forward_cpu_gemm is so that we can skip the im2col if
+ // we just called weight_cpu_gemm with the same input.
+ void forward_cpu_gemm(const Dtype* input, const Dtype* weights,
+ Dtype* output, bool skip_im2col = false);
+ void forward_cpu_bias(Dtype* output, const Dtype* bias);
+ void backward_cpu_gemm(const Dtype* input, const Dtype* weights,
+ Dtype* output);
+ void weight_cpu_gemm(const Dtype* input, const Dtype* output, Dtype*
+ weights);
+ void backward_cpu_bias(Dtype* bias, const Dtype* input);
+
+#ifndef CPU_ONLY
+ void forward_gpu_gemm(const Dtype* col_input, const Dtype* weights,
+ Dtype* output, bool skip_im2col = false);
+ void forward_gpu_bias(Dtype* output, const Dtype* bias);
+ void backward_gpu_gemm(const Dtype* input, const Dtype* weights,
+ Dtype* col_output);
+ void weight_gpu_gemm(const Dtype* col_input, const Dtype* output, Dtype*
+ weights);
+ void backward_gpu_bias(Dtype* bias, const Dtype* input);
+#endif
+
+ /// @brief The spatial dimensions of the input.
+ inline int input_shape(int i) {
+ return (*bottom_shape_)[channel_axis_ + i];
+ }
+ // reverse_dimensions should return true iff we are implementing deconv, so
+ // that conv helpers know which dimensions are which.
+ virtual bool reverse_dimensions() = 0;
+ // Compute height_out_ and width_out_ from other parameters.
+ virtual void compute_output_shape() = 0;
+
+ /// @brief The spatial dimensions of a filter kernel.
+ Blob<int> kernel_shape_;
+ /// @brief The spatial dimensions of the stride.
+ Blob<int> stride_;
+ /// @brief The spatial dimensions of the padding.
+ Blob<int> pad_;
+ /// @brief The spatial dimensions of the convolution input.
+ Blob<int> conv_input_shape_;
+ /// @brief The spatial dimensions of the col_buffer.
+ vector<int> col_buffer_shape_;
+ /// @brief The spatial dimensions of the output.
+ vector<int> output_shape_;
+ const vector<int>* bottom_shape_;
+
+ int num_spatial_axes_;
+ int bottom_dim_;
+ int top_dim_;
+
+ int channel_axis_;
+ int num_;
+ int channels_;
+ int group_;
+ int out_spatial_dim_;
+ int weight_offset_;
+ int num_output_;
+ bool bias_term_;
+ bool is_1x1_;
+ bool force_nd_im2col_;
+
+ private:
+ // wrap im2col/col2im so we don't have to remember the (long) argument lists
+ inline void conv_im2col_cpu(const Dtype* data, Dtype* col_buff) {
+ if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
+ im2col_cpu(data, conv_in_channels_,
+ conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
+ kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
+ pad_.cpu_data()[0], pad_.cpu_data()[1],
+ stride_.cpu_data()[0], stride_.cpu_data()[1], col_buff);
+ } else {
+ im2col_nd_cpu(data, num_spatial_axes_, conv_input_shape_.cpu_data(),
+ col_buffer_shape_.data(), kernel_shape_.cpu_data(),
+ pad_.cpu_data(), stride_.cpu_data(), col_buff);
+ }
+ }
+ inline void conv_col2im_cpu(const Dtype* col_buff, Dtype* data) {
+ if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
+ col2im_cpu(col_buff, conv_in_channels_,
+ conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
+ kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
+ pad_.cpu_data()[0], pad_.cpu_data()[1],
+ stride_.cpu_data()[0], stride_.cpu_data()[1], data);
+ } else {
+ col2im_nd_cpu(col_buff, num_spatial_axes_, conv_input_shape_.cpu_data(),
+ col_buffer_shape_.data(), kernel_shape_.cpu_data(),
+ pad_.cpu_data(), stride_.cpu_data(), data);
+ }
+ }
+#ifndef CPU_ONLY
+ inline void conv_im2col_gpu(const Dtype* data, Dtype* col_buff) {
+ if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
+ im2col_gpu(data, conv_in_channels_,
+ conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
+ kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
+ pad_.cpu_data()[0], pad_.cpu_data()[1],
+ stride_.cpu_data()[0], stride_.cpu_data()[1], col_buff);
+ } else {
+ im2col_nd_gpu(data, num_spatial_axes_, num_kernels_im2col_,
+ conv_input_shape_.gpu_data(), col_buffer_.gpu_shape(),
+ kernel_shape_.gpu_data(), pad_.gpu_data(),
+ stride_.gpu_data(), col_buff);
+ }
+ }
+ inline void conv_col2im_gpu(const Dtype* col_buff, Dtype* data) {
+ if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
+ col2im_gpu(col_buff, conv_in_channels_,
+ conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
+ kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
+ pad_.cpu_data()[0], pad_.cpu_data()[1],
+ stride_.cpu_data()[0], stride_.cpu_data()[1], data);
+ } else {
+ col2im_nd_gpu(col_buff, num_spatial_axes_, num_kernels_col2im_,
+ conv_input_shape_.gpu_data(), col_buffer_.gpu_shape(),
+ kernel_shape_.gpu_data(), pad_.gpu_data(), stride_.gpu_data(),
+ data);
+ }
+ }
+#endif
+
+ int num_kernels_im2col_;
+ int num_kernels_col2im_;
+ int conv_out_channels_;
+ int conv_in_channels_;
+ int conv_out_spatial_dim_;
+ int kernel_dim_;
+ int col_offset_;
+ int output_offset_;
+
+ Blob<Dtype> col_buffer_;
+ Blob<Dtype> bias_multiplier_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_BASE_CONVOLUTION_LAYER_HPP_
diff --git a/include/caffe/layers/base_data_layer.hpp b/include/caffe/layers/base_data_layer.hpp
new file mode 100644
index 0000000..2c49b73
--- /dev/null
+++ b/include/caffe/layers/base_data_layer.hpp
@@ -0,0 +1,86 @@
+#ifndef CAFFE_DATA_LAYERS_HPP_
+#define CAFFE_DATA_LAYERS_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/data_transformer.hpp"
+#include "caffe/internal_thread.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+#include "caffe/util/blocking_queue.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Provides base for data layers that feed blobs to the Net.
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class BaseDataLayer : public Layer<Dtype> {
+ public:
+ explicit BaseDataLayer(const LayerParameter& param);
+ // LayerSetUp: implements common data layer setup functionality, and calls
+ // DataLayerSetUp to do special data layer setup for individual layer types.
+ // This method may not be overridden except by the BasePrefetchingDataLayer.
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ // Data layers should be shared by multiple solvers in parallel
+ virtual inline bool ShareInParallel() const { return true; }
+ virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+ // Data layers have no bottoms, so reshaping is trivial.
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+
+ protected:
+ TransformationParameter transform_param_;
+ shared_ptr<DataTransformer<Dtype> > data_transformer_;
+ bool output_labels_;
+};
+
+template <typename Dtype>
+class Batch {
+ public:
+ Blob<Dtype> data_, label_;
+};
+
+template <typename Dtype>
+class BasePrefetchingDataLayer :
+ public BaseDataLayer<Dtype>, public InternalThread {
+ public:
+ explicit BasePrefetchingDataLayer(const LayerParameter& param);
+ // LayerSetUp: implements common data layer setup functionality, and calls
+ // DataLayerSetUp to do special data layer setup for individual layer types.
+ // This method may not be overridden.
+ void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ // Prefetches batches (asynchronously if to GPU memory)
+ static const int PREFETCH_COUNT = 3;
+
+ protected:
+ virtual void InternalThreadEntry();
+ virtual void load_batch(Batch<Dtype>* batch) = 0;
+
+ Batch<Dtype> prefetch_[PREFETCH_COUNT];
+ BlockingQueue<Batch<Dtype>*> prefetch_free_;
+ BlockingQueue<Batch<Dtype>*> prefetch_full_;
+
+ Blob<Dtype> transformed_data_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_DATA_LAYERS_HPP_
diff --git a/include/caffe/layers/batch_norm_layer.hpp b/include/caffe/layers/batch_norm_layer.hpp
new file mode 100644
index 0000000..9b2d512
--- /dev/null
+++ b/include/caffe/layers/batch_norm_layer.hpp
@@ -0,0 +1,81 @@
+#ifndef CAFFE_BATCHNORM_LAYER_HPP_
+#define CAFFE_BATCHNORM_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Normalizes the input to have 0-mean and/or unit (1) variance across
+ * the batch.
+ *
+ * This layer computes Batch Normalization described in [1]. For
+ * each channel in the data (i.e. axis 1), it subtracts the mean and divides
+ * by the variance, where both statistics are computed across both spatial
+ * dimensions and across the different examples in the batch.
+ *
+ * By default, during training time, the network is computing global mean/
+ * variance statistics via a running average, which is then used at test
+ * time to allow deterministic outputs for each input. You can manually
+ * toggle whether the network is accumulating or using the statistics via the
+ * use_global_stats option. IMPORTANT: for this feature to work, you MUST
+ * set the learning rate to zero for all three parameter blobs, i.e.,
+ * param {lr_mult: 0} three times in the layer definition.
+ *
+ * Note that the original paper also included a per-channel learned bias and
+ * scaling factor. It is possible (though a bit cumbersome) to implement
+ * this in caffe using a single-channel DummyDataLayer filled with zeros,
+ * followed by a Convolution layer with output the same size as the current.
+ * This produces a channel-specific value that can be added or multiplied by
+ * the BatchNorm layer's output.
+ *
+ * [1] S. Ioffe and C. Szegedy, "Batch Normalization: Accelerating Deep Network
+ * Training by Reducing Internal Covariate Shift." arXiv preprint
+ * arXiv:1502.03167 (2015).
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class BatchNormLayer : public Layer<Dtype> {
+ public:
+ explicit BatchNormLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "BatchNorm"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ Blob<Dtype> mean_, variance_, temp_, x_norm_;
+ bool use_global_stats_;
+ Dtype moving_average_fraction_;
+ int channels_;
+ Dtype eps_;
+
+ // extra temporarary variables is used to carry out sums/broadcasting
+ // using BLAS
+ Blob<Dtype> batch_sum_multiplier_;
+ Blob<Dtype> num_by_chans_;
+ Blob<Dtype> spatial_sum_multiplier_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_BATCHNORM_LAYER_HPP_
diff --git a/include/caffe/layers/batch_reindex_layer.hpp b/include/caffe/layers/batch_reindex_layer.hpp
new file mode 100644
index 0000000..ebb3a56
--- /dev/null
+++ b/include/caffe/layers/batch_reindex_layer.hpp
@@ -0,0 +1,83 @@
+#ifndef CAFFE_BATCHREINDEX_LAYER_HPP_
+#define CAFFE_BATCHREINDEX_LAYER_HPP_
+
+#include <utility>
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Index into the input blob along its first axis.
+ *
+ * This layer can be used to select, reorder, and even replicate examples in a
+ * batch. The second blob is cast to int and treated as an index into the
+ * first axis of the first blob.
+ */
+template <typename Dtype>
+class BatchReindexLayer : public Layer<Dtype> {
+ public:
+ explicit BatchReindexLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "BatchReindex"; }
+ virtual inline int ExactNumBottomBlobs() const { return 2; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 2+)
+ * -# @f$ (N \times ...) @f$
+ * the inputs @f$ x_1 @f$
+ * -# @f$ (M) @f$
+ * the inputs @f$ x_2 @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (M \times ...) @f$:
+ * the reindexed array @f$
+ * y = x_1[x_2]
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the reordered input.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient
+ * with respect to the outputs
+ * -# @f$ (M \times ...) @f$:
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to concatenated outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 2):
+ * - @f$ \frac{\partial E}{\partial y} @f$ is de-indexed (summing where
+ * required) back to the input x_1
+ * - This layer cannot backprop to x_2, i.e. propagate_down[1] must be
+ * false.
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ private:
+ struct pair_sort_first {
+ bool operator()(const std::pair<int, int> &left,
+ const std::pair<int, int> &right) {
+ return left.first < right.first;
+ }
+ };
+ void check_batch_reindex(int initial_num, int final_num,
+ const Dtype* ridx_data);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_BATCHREINDEX_LAYER_HPP_
diff --git a/include/caffe/layers/bnll_layer.hpp b/include/caffe/layers/bnll_layer.hpp
new file mode 100644
index 0000000..be07c74
--- /dev/null
+++ b/include/caffe/layers/bnll_layer.hpp
@@ -0,0 +1,70 @@
+#ifndef CAFFE_BNLL_LAYER_HPP_
+#define CAFFE_BNLL_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes @f$ y = x + \log(1 + \exp(-x)) @f$ if @f$ x > 0 @f$;
+ * @f$ y = \log(1 + \exp(x)) @f$ otherwise.
+ *
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = \left\{
+ * \begin{array}{ll}
+ * x + \log(1 + \exp(-x)) & \mbox{if } x > 0 \\
+ * \log(1 + \exp(x)) & \mbox{otherwise}
+ * \end{array} \right.
+ * @f$
+ */
+template <typename Dtype>
+class BNLLLayer : public NeuronLayer<Dtype> {
+ public:
+ explicit BNLLLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+
+ virtual inline const char* type() const { return "BNLL"; }
+
+ protected:
+ /// @copydoc BNLLLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the BNLL inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x}
+ * @f$ if propagate_down[0]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_BNLL_LAYER_HPP_
diff --git a/include/caffe/layers/concat_layer.hpp b/include/caffe/layers/concat_layer.hpp
new file mode 100644
index 0000000..a157024
--- /dev/null
+++ b/include/caffe/layers/concat_layer.hpp
@@ -0,0 +1,87 @@
+#ifndef CAFFE_CONCAT_LAYER_HPP_
+#define CAFFE_CONCAT_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Takes at least two Blob%s and concatenates them along either the num
+ * or channel dimension, outputting the result.
+ */
+template <typename Dtype>
+class ConcatLayer : public Layer<Dtype> {
+ public:
+ explicit ConcatLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Concat"; }
+ virtual inline int MinBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 2+)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x_1 @f$
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x_2 @f$
+ * -# ...
+ * - K @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x_K @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (KN \times C \times H \times W) @f$ if axis == 0, or
+ * @f$ (N \times KC \times H \times W) @f$ if axis == 1:
+ * the concatenated output @f$
+ * y = [\begin{array}{cccc} x_1 & x_2 & ... & x_K \end{array}]
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the concatenate inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (KN \times C \times H \times W) @f$ if axis == 0, or
+ * @f$ (N \times KC \times H \times W) @f$ if axis == 1:
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to concatenated outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length K), into which the top gradient
+ * @f$ \frac{\partial E}{\partial y} @f$ is deconcatenated back to the
+ * inputs @f$
+ * \left[ \begin{array}{cccc}
+ * \frac{\partial E}{\partial x_1} &
+ * \frac{\partial E}{\partial x_2} &
+ * ... &
+ * \frac{\partial E}{\partial x_K}
+ * \end{array} \right] =
+ * \frac{\partial E}{\partial y}
+ * @f$
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int count_;
+ int num_concats_;
+ int concat_input_size_;
+ int concat_axis_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_CONCAT_LAYER_HPP_
diff --git a/include/caffe/layers/contrastive_loss_layer.hpp b/include/caffe/layers/contrastive_loss_layer.hpp
new file mode 100644
index 0000000..e890afb
--- /dev/null
+++ b/include/caffe/layers/contrastive_loss_layer.hpp
@@ -0,0 +1,101 @@
+#ifndef CAFFE_CONTRASTIVE_LOSS_LAYER_HPP_
+#define CAFFE_CONTRASTIVE_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes the contrastive loss @f$
+ * E = \frac{1}{2N} \sum\limits_{n=1}^N \left(y\right) d^2 +
+ * \left(1-y\right) \max \left(margin-d, 0\right)^2
+ * @f$ where @f$
+ * d = \left| \left| a_n - b_n \right| \right|_2 @f$. This can be
+ * used to train siamese networks.
+ *
+ * @param bottom input Blob vector (length 3)
+ * -# @f$ (N \times C \times 1 \times 1) @f$
+ * the features @f$ a \in [-\infty, +\infty]@f$
+ * -# @f$ (N \times C \times 1 \times 1) @f$
+ * the features @f$ b \in [-\infty, +\infty]@f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the binary similarity @f$ s \in [0, 1]@f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed contrastive loss: @f$ E =
+ * \frac{1}{2N} \sum\limits_{n=1}^N \left(y\right) d^2 +
+ * \left(1-y\right) \max \left(margin-d, 0\right)^2
+ * @f$ where @f$
+ * d = \left| \left| a_n - b_n \right| \right|_2 @f$.
+ * This can be used to train siamese networks.
+ */
+template <typename Dtype>
+class ContrastiveLossLayer : public LossLayer<Dtype> {
+ public:
+ explicit ContrastiveLossLayer(const LayerParameter& param)
+ : LossLayer<Dtype>(param), diff_() {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline int ExactNumBottomBlobs() const { return 3; }
+ virtual inline const char* type() const { return "ContrastiveLoss"; }
+ /**
+ * Unlike most loss layers, in the ContrastiveLossLayer we can backpropagate
+ * to the first two inputs.
+ */
+ virtual inline bool AllowForceBackward(const int bottom_index) const {
+ return bottom_index != 2;
+ }
+
+ protected:
+ /// @copydoc ContrastiveLossLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the Contrastive error gradient w.r.t. the inputs.
+ *
+ * Computes the gradients with respect to the two input vectors (bottom[0] and
+ * bottom[1]), but not the similarity label (bottom[2]).
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
+ * as @f$ \lambda @f$ is the coefficient of this layer's output
+ * @f$\ell_i at f$ in the overall Net loss
+ * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
+ * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
+ * (*Assuming that this top Blob is not used as a bottom (input) by any
+ * other layer of the Net.)
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times 1 \times 1) @f$
+ * the features @f$a at f$; Backward fills their diff with
+ * gradients if propagate_down[0]
+ * -# @f$ (N \times C \times 1 \times 1) @f$
+ * the features @f$b at f$; Backward fills their diff with gradients if
+ * propagate_down[1]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ Blob<Dtype> diff_; // cached for backward pass
+ Blob<Dtype> dist_sq_; // cached for backward pass
+ Blob<Dtype> diff_sq_; // tmp storage for gpu forward pass
+ Blob<Dtype> summer_vec_; // tmp storage for gpu forward pass
+};
+
+} // namespace caffe
+
+#endif // CAFFE_CONTRASTIVE_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/conv_layer.hpp b/include/caffe/layers/conv_layer.hpp
new file mode 100644
index 0000000..1557476
--- /dev/null
+++ b/include/caffe/layers/conv_layer.hpp
@@ -0,0 +1,81 @@
+#ifndef CAFFE_CONV_LAYER_HPP_
+#define CAFFE_CONV_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/base_conv_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Convolves the input image with a bank of learned filters,
+ * and (optionally) adds biases.
+ *
+ * Caffe convolves by reduction to matrix multiplication. This achieves
+ * high-throughput and generality of input and filter dimensions but comes at
+ * the cost of memory for matrices. This makes use of efficiency in BLAS.
+ *
+ * The input is "im2col" transformed to a channel K' x H x W data matrix
+ * for multiplication with the N x K' x H x W filter matrix to yield a
+ * N' x H x W output matrix that is then "col2im" restored. K' is the
+ * input channel * kernel height * kernel width dimension of the unrolled
+ * inputs so that the im2col matrix has a column for each input region to
+ * be filtered. col2im restores the output spatial structure by rolling up
+ * the output channel N' columns of the output matrix.
+ */
+template <typename Dtype>
+class ConvolutionLayer : public BaseConvolutionLayer<Dtype> {
+ public:
+ /**
+ * @param param provides ConvolutionParameter convolution_param,
+ * with ConvolutionLayer options:
+ * - num_output. The number of filters.
+ * - kernel_size / kernel_h / kernel_w. The filter dimensions, given by
+ * kernel_size for square filters or kernel_h and kernel_w for rectangular
+ * filters.
+ * - stride / stride_h / stride_w (\b optional, default 1). The filter
+ * stride, given by stride_size for equal dimensions or stride_h and stride_w
+ * for different strides. By default the convolution is dense with stride 1.
+ * - pad / pad_h / pad_w (\b optional, default 0). The zero-padding for
+ * convolution, given by pad for equal dimensions or pad_h and pad_w for
+ * different padding. Input padding is computed implicitly instead of
+ * actually padding.
+ * - group (\b optional, default 1). The number of filter groups. Group
+ * convolution is a method for reducing parameterization by selectively
+ * connecting input and output channels. The input and output channel dimensions must be divisible
+ * by the number of groups. For group @f$ \geq 1 @f$, the
+ * convolutional filters' input and output channels are separated s.t. each
+ * group takes 1 / group of the input channels and makes 1 / group of the
+ * output channels. Concretely 4 input channels, 8 output channels, and
+ * 2 groups separate input channels 1-2 and output channels 1-4 into the
+ * first group and input channels 3-4 and output channels 5-8 into the second
+ * group.
+ * - bias_term (\b optional, default true). Whether to have a bias.
+ * - engine: convolution has CAFFE (matrix multiplication) and CUDNN (library
+ * kernels + stream parallelism) engines.
+ */
+ explicit ConvolutionLayer(const LayerParameter& param)
+ : BaseConvolutionLayer<Dtype>(param) {}
+
+ virtual inline const char* type() const { return "Convolution"; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual inline bool reverse_dimensions() { return false; }
+ virtual void compute_output_shape();
+};
+
+} // namespace caffe
+
+#endif // CAFFE_CONV_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_conv_layer.hpp b/include/caffe/layers/cudnn_conv_layer.hpp
new file mode 100644
index 0000000..31fe49a
--- /dev/null
+++ b/include/caffe/layers/cudnn_conv_layer.hpp
@@ -0,0 +1,72 @@
+#ifndef CAFFE_CUDNN_CONV_LAYER_HPP_
+#define CAFFE_CUDNN_CONV_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/conv_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+/*
+ * @brief cuDNN implementation of ConvolutionLayer.
+ * Fallback to ConvolutionLayer for CPU mode.
+ *
+ * cuDNN accelerates convolution through forward kernels for filtering and bias
+ * plus backward kernels for the gradient w.r.t. the filters, biases, and
+ * inputs. Caffe + cuDNN further speeds up the computation through forward
+ * parallelism across groups and backward parallelism across gradients.
+ *
+ * The CUDNN engine does not have memory overhead for matrix buffers. For many
+ * input and filter regimes the CUDNN engine is faster than the CAFFE engine,
+ * but for fully-convolutional models and large inputs the CAFFE engine can be
+ * faster as long as it fits in memory.
+*/
+template <typename Dtype>
+class CuDNNConvolutionLayer : public ConvolutionLayer<Dtype> {
+ public:
+ explicit CuDNNConvolutionLayer(const LayerParameter& param)
+ : ConvolutionLayer<Dtype>(param), handles_setup_(false) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNConvolutionLayer();
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t* handle_;
+ cudaStream_t* stream_;
+
+ // algorithms for forward and backwards convolutions
+ cudnnConvolutionFwdAlgo_t *fwd_algo_;
+ cudnnConvolutionBwdFilterAlgo_t *bwd_filter_algo_;
+ cudnnConvolutionBwdDataAlgo_t *bwd_data_algo_;
+
+ vector<cudnnTensorDescriptor_t> bottom_descs_, top_descs_;
+ cudnnTensorDescriptor_t bias_desc_;
+ cudnnFilterDescriptor_t filter_desc_;
+ vector<cudnnConvolutionDescriptor_t> conv_descs_;
+ int bottom_offset_, top_offset_, bias_offset_;
+
+ size_t *workspace_fwd_sizes_;
+ size_t *workspace_bwd_data_sizes_;
+ size_t *workspace_bwd_filter_sizes_;
+ size_t workspaceSizeInBytes; // size of underlying storage
+ void *workspaceData; // underlying storage
+ void **workspace; // aliases into workspaceData
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_CONV_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_lcn_layer.hpp b/include/caffe/layers/cudnn_lcn_layer.hpp
new file mode 100644
index 0000000..74cf477
--- /dev/null
+++ b/include/caffe/layers/cudnn_lcn_layer.hpp
@@ -0,0 +1,49 @@
+#ifndef CAFFE_CUDNN_LCN_LAYER_HPP_
+#define CAFFE_CUDNN_LCN_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/lrn_layer.hpp"
+#include "caffe/layers/power_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+template <typename Dtype>
+class CuDNNLCNLayer : public LRNLayer<Dtype> {
+ public:
+ explicit CuDNNLCNLayer(const LayerParameter& param)
+ : LRNLayer<Dtype>(param), handles_setup_(false), tempDataSize(0),
+ tempData1(NULL), tempData2(NULL) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNLCNLayer();
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t handle_;
+ cudnnLRNDescriptor_t norm_desc_;
+ cudnnTensorDescriptor_t bottom_desc_, top_desc_;
+
+ int size_, pre_pad_;
+ Dtype alpha_, beta_, k_;
+
+ size_t tempDataSize;
+ void *tempData1, *tempData2;
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_LCN_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_lrn_layer.hpp b/include/caffe/layers/cudnn_lrn_layer.hpp
new file mode 100644
index 0000000..000ccc3
--- /dev/null
+++ b/include/caffe/layers/cudnn_lrn_layer.hpp
@@ -0,0 +1,44 @@
+#ifndef CAFFE_CUDNN_LRN_LAYER_HPP_
+#define CAFFE_CUDNN_LRN_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/lrn_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+template <typename Dtype>
+class CuDNNLRNLayer : public LRNLayer<Dtype> {
+ public:
+ explicit CuDNNLRNLayer(const LayerParameter& param)
+ : LRNLayer<Dtype>(param), handles_setup_(false) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNLRNLayer();
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t handle_;
+ cudnnLRNDescriptor_t norm_desc_;
+ cudnnTensorDescriptor_t bottom_desc_, top_desc_;
+
+ int size_;
+ Dtype alpha_, beta_, k_;
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_LRN_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_pooling_layer.hpp b/include/caffe/layers/cudnn_pooling_layer.hpp
new file mode 100644
index 0000000..6d0db47
--- /dev/null
+++ b/include/caffe/layers/cudnn_pooling_layer.hpp
@@ -0,0 +1,49 @@
+#ifndef CAFFE_CUDNN_POOLING_LAYER_HPP_
+#define CAFFE_CUDNN_POOLING_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/pooling_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+/*
+ * @brief cuDNN implementation of PoolingLayer.
+ * Fallback to PoolingLayer for CPU mode.
+*/
+template <typename Dtype>
+class CuDNNPoolingLayer : public PoolingLayer<Dtype> {
+ public:
+ explicit CuDNNPoolingLayer(const LayerParameter& param)
+ : PoolingLayer<Dtype>(param), handles_setup_(false) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNPoolingLayer();
+ // Currently, cuDNN does not support the extra top blob.
+ virtual inline int MinTopBlobs() const { return -1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t handle_;
+ cudnnTensorDescriptor_t bottom_desc_, top_desc_;
+ cudnnPoolingDescriptor_t pooling_desc_;
+ cudnnPoolingMode_t mode_;
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_POOLING_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_relu_layer.hpp b/include/caffe/layers/cudnn_relu_layer.hpp
new file mode 100644
index 0000000..e01f568
--- /dev/null
+++ b/include/caffe/layers/cudnn_relu_layer.hpp
@@ -0,0 +1,45 @@
+#ifndef CAFFE_CUDNN_RELU_LAYER_HPP_
+#define CAFFE_CUDNN_RELU_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+#include "caffe/layers/relu_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+/**
+ * @brief CuDNN acceleration of ReLULayer.
+ */
+template <typename Dtype>
+class CuDNNReLULayer : public ReLULayer<Dtype> {
+ public:
+ explicit CuDNNReLULayer(const LayerParameter& param)
+ : ReLULayer<Dtype>(param), handles_setup_(false) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNReLULayer();
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t handle_;
+ cudnnTensorDescriptor_t bottom_desc_;
+ cudnnTensorDescriptor_t top_desc_;
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_RELU_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_sigmoid_layer.hpp b/include/caffe/layers/cudnn_sigmoid_layer.hpp
new file mode 100644
index 0000000..9c59795
--- /dev/null
+++ b/include/caffe/layers/cudnn_sigmoid_layer.hpp
@@ -0,0 +1,45 @@
+#ifndef CAFFE_CUDNN_SIGMOID_LAYER_HPP_
+#define CAFFE_CUDNN_SIGMOID_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+#include "caffe/layers/sigmoid_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+/**
+ * @brief CuDNN acceleration of SigmoidLayer.
+ */
+template <typename Dtype>
+class CuDNNSigmoidLayer : public SigmoidLayer<Dtype> {
+ public:
+ explicit CuDNNSigmoidLayer(const LayerParameter& param)
+ : SigmoidLayer<Dtype>(param), handles_setup_(false) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNSigmoidLayer();
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t handle_;
+ cudnnTensorDescriptor_t bottom_desc_;
+ cudnnTensorDescriptor_t top_desc_;
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_SIGMOID_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_softmax_layer.hpp b/include/caffe/layers/cudnn_softmax_layer.hpp
new file mode 100644
index 0000000..174368e
--- /dev/null
+++ b/include/caffe/layers/cudnn_softmax_layer.hpp
@@ -0,0 +1,45 @@
+#ifndef CAFFE_CUDNN_SOFTMAX_LAYER_HPP_
+#define CAFFE_CUDNN_SOFTMAX_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/softmax_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+/**
+ * @brief cuDNN implementation of SoftmaxLayer.
+ * Fallback to SoftmaxLayer for CPU mode.
+ */
+template <typename Dtype>
+class CuDNNSoftmaxLayer : public SoftmaxLayer<Dtype> {
+ public:
+ explicit CuDNNSoftmaxLayer(const LayerParameter& param)
+ : SoftmaxLayer<Dtype>(param), handles_setup_(false) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNSoftmaxLayer();
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t handle_;
+ cudnnTensorDescriptor_t bottom_desc_;
+ cudnnTensorDescriptor_t top_desc_;
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_SOFTMAX_LAYER_HPP_
diff --git a/include/caffe/layers/cudnn_tanh_layer.hpp b/include/caffe/layers/cudnn_tanh_layer.hpp
new file mode 100644
index 0000000..c0f0053
--- /dev/null
+++ b/include/caffe/layers/cudnn_tanh_layer.hpp
@@ -0,0 +1,45 @@
+#ifndef CAFFE_CUDNN_TANH_LAYER_HPP_
+#define CAFFE_CUDNN_TANH_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+#include "caffe/layers/tanh_layer.hpp"
+
+namespace caffe {
+
+#ifdef USE_CUDNN
+/**
+ * @brief CuDNN acceleration of TanHLayer.
+ */
+template <typename Dtype>
+class CuDNNTanHLayer : public TanHLayer<Dtype> {
+ public:
+ explicit CuDNNTanHLayer(const LayerParameter& param)
+ : TanHLayer<Dtype>(param), handles_setup_(false) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual ~CuDNNTanHLayer();
+
+ protected:
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool handles_setup_;
+ cudnnHandle_t handle_;
+ cudnnTensorDescriptor_t bottom_desc_;
+ cudnnTensorDescriptor_t top_desc_;
+};
+#endif
+
+} // namespace caffe
+
+#endif // CAFFE_CUDNN_TANH_LAYER_HPP_
diff --git a/include/caffe/layers/data_layer.hpp b/include/caffe/layers/data_layer.hpp
new file mode 100644
index 0000000..6c36179
--- /dev/null
+++ b/include/caffe/layers/data_layer.hpp
@@ -0,0 +1,39 @@
+#ifndef CAFFE_DATA_LAYER_HPP_
+#define CAFFE_DATA_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/data_reader.hpp"
+#include "caffe/data_transformer.hpp"
+#include "caffe/internal_thread.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/layers/base_data_layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+#include "caffe/util/db.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+class DataLayer : public BasePrefetchingDataLayer<Dtype> {
+ public:
+ explicit DataLayer(const LayerParameter& param);
+ virtual ~DataLayer();
+ virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ // DataLayer uses DataReader instead for sharing for parallelism
+ virtual inline bool ShareInParallel() const { return false; }
+ virtual inline const char* type() const { return "Data"; }
+ virtual inline int ExactNumBottomBlobs() const { return 0; }
+ virtual inline int MinTopBlobs() const { return 1; }
+ virtual inline int MaxTopBlobs() const { return 2; }
+
+ protected:
+ virtual void load_batch(Batch<Dtype>* batch);
+
+ DataReader reader_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_DATA_LAYER_HPP_
diff --git a/include/caffe/layers/deconv_layer.hpp b/include/caffe/layers/deconv_layer.hpp
new file mode 100644
index 0000000..23ae887
--- /dev/null
+++ b/include/caffe/layers/deconv_layer.hpp
@@ -0,0 +1,51 @@
+#ifndef CAFFE_DECONV_LAYER_HPP_
+#define CAFFE_DECONV_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/base_conv_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Convolve the input with a bank of learned filters, and (optionally)
+ * add biases, treating filters and convolution parameters in the
+ * opposite sense as ConvolutionLayer.
+ *
+ * ConvolutionLayer computes each output value by dotting an input window with
+ * a filter; DeconvolutionLayer multiplies each input value by a filter
+ * elementwise, and sums over the resulting output windows. In other words,
+ * DeconvolutionLayer is ConvolutionLayer with the forward and backward passes
+ * reversed. DeconvolutionLayer reuses ConvolutionParameter for its
+ * parameters, but they take the opposite sense as in ConvolutionLayer (so
+ * padding is removed from the output rather than added to the input, and
+ * stride results in upsampling rather than downsampling).
+ */
+template <typename Dtype>
+class DeconvolutionLayer : public BaseConvolutionLayer<Dtype> {
+ public:
+ explicit DeconvolutionLayer(const LayerParameter& param)
+ : BaseConvolutionLayer<Dtype>(param) {}
+
+ virtual inline const char* type() const { return "Deconvolution"; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual inline bool reverse_dimensions() { return true; }
+ virtual void compute_output_shape();
+};
+
+} // namespace caffe
+
+#endif // CAFFE_DECONV_LAYER_HPP_
diff --git a/include/caffe/layers/dropout_layer.hpp b/include/caffe/layers/dropout_layer.hpp
new file mode 100644
index 0000000..e83143b
--- /dev/null
+++ b/include/caffe/layers/dropout_layer.hpp
@@ -0,0 +1,80 @@
+#ifndef CAFFE_DROPOUT_LAYER_HPP_
+#define CAFFE_DROPOUT_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief During training only, sets a random portion of @f$x at f$ to 0, adjusting
+ * the rest of the vector magnitude accordingly.
+ *
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$ y = |x| @f$
+ */
+template <typename Dtype>
+class DropoutLayer : public NeuronLayer<Dtype> {
+ public:
+ /**
+ * @param param provides DropoutParameter dropout_param,
+ * with DropoutLayer options:
+ * - dropout_ratio (\b optional, default 0.5).
+ * Sets the probability @f$ p @f$ that any given unit is dropped.
+ */
+ explicit DropoutLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Dropout"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs. At training time, we have @f$
+ * y_{\mbox{train}} = \left\{
+ * \begin{array}{ll}
+ * \frac{x}{1 - p} & \mbox{if } u > p \\
+ * 0 & \mbox{otherwise}
+ * \end{array} \right.
+ * @f$, where @f$ u \sim U(0, 1)@f$ is generated independently for each
+ * input at each iteration. At test time, we simply have
+ * @f$ y_{\mbox{test}} = \mathbb{E}[y_{\mbox{train}}] = x @f$.
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ /// when divided by UINT_MAX, the randomly generated values @f$u\sim U(0,1)@f$
+ Blob<unsigned int> rand_vec_;
+ /// the probability @f$ p @f$ of dropping any input
+ Dtype threshold_;
+ /// the scale for undropped inputs at train time @f$ 1 / (1 - p) @f$
+ Dtype scale_;
+ unsigned int uint_thres_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_DROPOUT_LAYER_HPP_
diff --git a/include/caffe/layers/dummy_data_layer.hpp b/include/caffe/layers/dummy_data_layer.hpp
new file mode 100644
index 0000000..4180f1d
--- /dev/null
+++ b/include/caffe/layers/dummy_data_layer.hpp
@@ -0,0 +1,49 @@
+#ifndef CAFFE_DUMMY_DATA_LAYER_HPP_
+#define CAFFE_DUMMY_DATA_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/filler.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Provides data to the Net generated by a Filler.
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class DummyDataLayer : public Layer<Dtype> {
+ public:
+ explicit DummyDataLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ // Data layers should be shared by multiple solvers in parallel
+ virtual inline bool ShareInParallel() const { return true; }
+ // Data layers have no bottoms, so reshaping is trivial.
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+
+ virtual inline const char* type() const { return "DummyData"; }
+ virtual inline int ExactNumBottomBlobs() const { return 0; }
+ virtual inline int MinTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+
+ vector<shared_ptr<Filler<Dtype> > > fillers_;
+ vector<bool> refill_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_DUMMY_DATA_LAYER_HPP_
diff --git a/include/caffe/layers/eltwise_layer.hpp b/include/caffe/layers/eltwise_layer.hpp
new file mode 100644
index 0000000..091de83
--- /dev/null
+++ b/include/caffe/layers/eltwise_layer.hpp
@@ -0,0 +1,51 @@
+#ifndef CAFFE_ELTWISE_LAYER_HPP_
+#define CAFFE_ELTWISE_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Compute elementwise operations, such as product and sum,
+ * along multiple input Blobs.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class EltwiseLayer : public Layer<Dtype> {
+ public:
+ explicit EltwiseLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Eltwise"; }
+ virtual inline int MinBottomBlobs() const { return 2; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ EltwiseParameter_EltwiseOp op_;
+ vector<Dtype> coeffs_;
+ Blob<int> max_idx_;
+
+ bool stable_prod_grad_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_ELTWISE_LAYER_HPP_
diff --git a/include/caffe/layers/embed_layer.hpp b/include/caffe/layers/embed_layer.hpp
new file mode 100644
index 0000000..36137a6
--- /dev/null
+++ b/include/caffe/layers/embed_layer.hpp
@@ -0,0 +1,52 @@
+#ifndef CAFFE_EMBED_LAYER_HPP_
+#define CAFFE_EMBED_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief A layer for learning "embeddings" of one-hot vector input.
+ * Equivalent to an InnerProductLayer with one-hot vectors as input, but
+ * for efficiency the input is the "hot" index of each column itself.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class EmbedLayer : public Layer<Dtype> {
+ public:
+ explicit EmbedLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Embed"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int M_;
+ int K_;
+ int N_;
+ bool bias_term_;
+ Blob<Dtype> bias_multiplier_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_EMBED_LAYER_HPP_
diff --git a/include/caffe/layers/euclidean_loss_layer.hpp b/include/caffe/layers/euclidean_loss_layer.hpp
new file mode 100644
index 0000000..f564569
--- /dev/null
+++ b/include/caffe/layers/euclidean_loss_layer.hpp
@@ -0,0 +1,107 @@
+#ifndef CAFFE_EUCLIDEAN_LOSS_LAYER_HPP_
+#define CAFFE_EUCLIDEAN_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes the Euclidean (L2) loss @f$
+ * E = \frac{1}{2N} \sum\limits_{n=1}^N \left| \left| \hat{y}_n - y_n
+ * \right| \right|_2^2 @f$ for real-valued regression tasks.
+ *
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ \hat{y} \in [-\infty, +\infty]@f$
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the targets @f$ y \in [-\infty, +\infty]@f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed Euclidean loss: @f$ E =
+ * \frac{1}{2n} \sum\limits_{n=1}^N \left| \left| \hat{y}_n - y_n
+ * \right| \right|_2^2 @f$
+ *
+ * This can be used for least-squares regression tasks. An InnerProductLayer
+ * input to a EuclideanLossLayer exactly formulates a linear least squares
+ * regression problem. With non-zero weight decay the problem becomes one of
+ * ridge regression -- see src/caffe/test/test_sgd_solver.cpp for a concrete
+ * example wherein we check that the gradients computed for a Net with exactly
+ * this structure match hand-computed gradient formulas for ridge regression.
+ *
+ * (Note: Caffe, and SGD in general, is certainly \b not the best way to solve
+ * linear least squares problems! We use it only as an instructive example.)
+ */
+template <typename Dtype>
+class EuclideanLossLayer : public LossLayer<Dtype> {
+ public:
+ explicit EuclideanLossLayer(const LayerParameter& param)
+ : LossLayer<Dtype>(param), diff_() {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "EuclideanLoss"; }
+ /**
+ * Unlike most loss layers, in the EuclideanLossLayer we can backpropagate
+ * to both inputs -- override to return true and always allow force_backward.
+ */
+ virtual inline bool AllowForceBackward(const int bottom_index) const {
+ return true;
+ }
+
+ protected:
+ /// @copydoc EuclideanLossLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the Euclidean error gradient w.r.t. the inputs.
+ *
+ * Unlike other children of LossLayer, EuclideanLossLayer \b can compute
+ * gradients with respect to the label inputs bottom[1] (but still only will
+ * if propagate_down[1] is set, due to being produced by learnable parameters
+ * or if force_backward is set). In fact, this layer is "commutative" -- the
+ * result is the same regardless of the order of the two bottoms.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
+ * as @f$ \lambda @f$ is the coefficient of this layer's output
+ * @f$\ell_i at f$ in the overall Net loss
+ * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
+ * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
+ * (*Assuming that this top Blob is not used as a bottom (input) by any
+ * other layer of the Net.)
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$\hat{y}@f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial \hat{y}} =
+ * \frac{1}{n} \sum\limits_{n=1}^N (\hat{y}_n - y_n)
+ * @f$ if propagate_down[0]
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the targets @f$y at f$; Backward fills their diff with gradients
+ * @f$ \frac{\partial E}{\partial y} =
+ * \frac{1}{n} \sum\limits_{n=1}^N (y_n - \hat{y}_n)
+ * @f$ if propagate_down[1]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ Blob<Dtype> diff_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_EUCLIDEAN_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/exp_layer.hpp b/include/caffe/layers/exp_layer.hpp
new file mode 100644
index 0000000..9fc8c39
--- /dev/null
+++ b/include/caffe/layers/exp_layer.hpp
@@ -0,0 +1,80 @@
+#ifndef CAFFE_EXP_LAYER_HPP_
+#define CAFFE_EXP_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes @f$ y = \gamma ^ {\alpha x + \beta} @f$,
+ * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$,
+ * and base @f$ \gamma @f$.
+ */
+template <typename Dtype>
+class ExpLayer : public NeuronLayer<Dtype> {
+ public:
+ /**
+ * @param param provides ExpParameter exp_param,
+ * with ExpLayer options:
+ * - scale (\b optional, default 1) the scale @f$ \alpha @f$
+ * - shift (\b optional, default 0) the shift @f$ \beta @f$
+ * - base (\b optional, default -1 for a value of @f$ e \approx 2.718 @f$)
+ * the base @f$ \gamma @f$
+ */
+ explicit ExpLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Exp"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = \gamma ^ {\alpha x + \beta}
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the exp inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x} =
+ * \frac{\partial E}{\partial y} y \alpha \log_e(gamma)
+ * @f$ if propagate_down[0]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ Dtype inner_scale_, outer_scale_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_EXP_LAYER_HPP_
diff --git a/include/caffe/layers/filter_layer.hpp b/include/caffe/layers/filter_layer.hpp
new file mode 100644
index 0000000..e040e66
--- /dev/null
+++ b/include/caffe/layers/filter_layer.hpp
@@ -0,0 +1,77 @@
+#ifndef CAFFE_FILTER_LAYER_HPP_
+#define CAFFE_FILTER_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Takes two+ Blobs, interprets last Blob as a selector and
+ * filter remaining Blobs accordingly with selector data (0 means that
+ * the corresponding item has to be filtered, non-zero means that corresponding
+ * item needs to stay).
+ */
+template <typename Dtype>
+class FilterLayer : public Layer<Dtype> {
+ public:
+ explicit FilterLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Filter"; }
+ virtual inline int MinBottomBlobs() const { return 2; }
+ virtual inline int MinTopBlobs() const { return 1; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 2+)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs to be filtered @f$ x_1 @f$
+ * -# ...
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs to be filtered @f$ x_K @f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the selector blob
+ * @param top output Blob vector (length 1+)
+ * -# @f$ (S \times C \times H \times W) @f$ ()
+ * the filtered output @f$ x_1 @f$
+ * where S is the number of items
+ * that haven't been filtered
+ * @f$ (S \times C \times H \times W) @f$
+ * the filtered output @f$ x_K @f$
+ * where S is the number of items
+ * that haven't been filtered
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the forwarded inputs.
+ *
+ * @param top output Blob vector (length 1+), providing the error gradient with
+ * respect to the outputs
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 2+), into which the top error
+ * gradient is copied
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool first_reshape_;
+ vector<int> indices_to_forward_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_FILTER_LAYER_HPP_
diff --git a/include/caffe/layers/flatten_layer.hpp b/include/caffe/layers/flatten_layer.hpp
new file mode 100644
index 0000000..e494bbb
--- /dev/null
+++ b/include/caffe/layers/flatten_layer.hpp
@@ -0,0 +1,61 @@
+#ifndef CAFFE_FLATTEN_LAYER_HPP_
+#define CAFFE_FLATTEN_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Reshapes the input Blob into flat vectors.
+ *
+ * Note: because this layer does not change the input values -- merely the
+ * dimensions -- it can simply copy the input. The copy happens "virtually"
+ * (thus taking effectively 0 real time) by setting, in Forward, the data
+ * pointer of the top Blob to that of the bottom Blob (see Blob::ShareData),
+ * and in Backward, the diff pointer of the bottom Blob to that of the top Blob
+ * (see Blob::ShareDiff).
+ */
+template <typename Dtype>
+class FlattenLayer : public Layer<Dtype> {
+ public:
+ explicit FlattenLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Flatten"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 2+)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times CHW \times 1 \times 1) @f$
+ * the outputs -- i.e., the (virtually) copied, flattened inputs
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the concatenate inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length K), into which the top error
+ * gradient is (virtually) copied
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_FLATTEN_LAYER_HPP_
diff --git a/include/caffe/layers/hdf5_data_layer.hpp b/include/caffe/layers/hdf5_data_layer.hpp
new file mode 100644
index 0000000..b04cf8e
--- /dev/null
+++ b/include/caffe/layers/hdf5_data_layer.hpp
@@ -0,0 +1,62 @@
+#ifndef CAFFE_HDF5_DATA_LAYER_HPP_
+#define CAFFE_HDF5_DATA_LAYER_HPP_
+
+#include "hdf5.h"
+
+#include <string>
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/base_data_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Provides data to the Net from HDF5 files.
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class HDF5DataLayer : public Layer<Dtype> {
+ public:
+ explicit HDF5DataLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual ~HDF5DataLayer();
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ // Data layers should be shared by multiple solvers in parallel
+ virtual inline bool ShareInParallel() const { return true; }
+ // Data layers have no bottoms, so reshaping is trivial.
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+
+ virtual inline const char* type() const { return "HDF5Data"; }
+ virtual inline int ExactNumBottomBlobs() const { return 0; }
+ virtual inline int MinTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+ virtual void LoadHDF5FileData(const char* filename);
+
+ std::vector<std::string> hdf_filenames_;
+ unsigned int num_files_;
+ unsigned int current_file_;
+ hsize_t current_row_;
+ std::vector<shared_ptr<Blob<Dtype> > > hdf_blobs_;
+ std::vector<unsigned int> data_permutation_;
+ std::vector<unsigned int> file_permutation_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_HDF5_DATA_LAYER_HPP_
diff --git a/include/caffe/layers/hdf5_output_layer.hpp b/include/caffe/layers/hdf5_output_layer.hpp
new file mode 100644
index 0000000..487d08f
--- /dev/null
+++ b/include/caffe/layers/hdf5_output_layer.hpp
@@ -0,0 +1,64 @@
+#ifndef CAFFE_HDF5_OUTPUT_LAYER_HPP_
+#define CAFFE_HDF5_OUTPUT_LAYER_HPP_
+
+#include "hdf5.h"
+
+#include <string>
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+#define HDF5_DATA_DATASET_NAME "data"
+#define HDF5_DATA_LABEL_NAME "label"
+
+/**
+ * @brief Write blobs to disk as HDF5 files.
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class HDF5OutputLayer : public Layer<Dtype> {
+ public:
+ explicit HDF5OutputLayer(const LayerParameter& param)
+ : Layer<Dtype>(param), file_opened_(false) {}
+ virtual ~HDF5OutputLayer();
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ // Data layers should be shared by multiple solvers in parallel
+ virtual inline bool ShareInParallel() const { return true; }
+ // Data layers have no bottoms, so reshaping is trivial.
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+
+ virtual inline const char* type() const { return "HDF5Output"; }
+ // TODO: no limit on the number of blobs
+ virtual inline int ExactNumBottomBlobs() const { return 2; }
+ virtual inline int ExactNumTopBlobs() const { return 0; }
+
+ inline std::string file_name() const { return file_name_; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void SaveBlobs();
+
+ bool file_opened_;
+ std::string file_name_;
+ hid_t file_id_;
+ Blob<Dtype> data_blob_;
+ Blob<Dtype> label_blob_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_HDF5_OUTPUT_LAYER_HPP_
diff --git a/include/caffe/layers/hinge_loss_layer.hpp b/include/caffe/layers/hinge_loss_layer.hpp
new file mode 100644
index 0000000..54e42bd
--- /dev/null
+++ b/include/caffe/layers/hinge_loss_layer.hpp
@@ -0,0 +1,104 @@
+#ifndef CAFFE_HINGE_LOSS_LAYER_HPP_
+#define CAFFE_HINGE_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes the hinge loss for a one-of-many classification task.
+ *
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ t @f$, a Blob with values in
+ * @f$ [-\infty, +\infty] @f$ indicating the predicted score for each of
+ * the @f$ K = CHW @f$ classes. In an SVM, @f$ t @f$ is the result of
+ * taking the inner product @f$ X^T W @f$ of the D-dimensional features
+ * @f$ X \in \mathcal{R}^{D \times N} @f$ and the learned hyperplane
+ * parameters @f$ W \in \mathcal{R}^{D \times K} @f$, so a Net with just
+ * an InnerProductLayer (with num_output = D) providing predictions to a
+ * HingeLossLayer and no other learnable parameters or losses is
+ * equivalent to an SVM.
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels @f$ l @f$, an integer-valued Blob with values
+ * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
+ * indicating the correct class label among the @f$ K @f$ classes
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed hinge loss: @f$ E =
+ * \frac{1}{N} \sum\limits_{n=1}^N \sum\limits_{k=1}^K
+ * [\max(0, 1 - \delta\{l_n = k\} t_{nk})] ^ p
+ * @f$, for the @f$ L^p @f$ norm
+ * (defaults to @f$ p = 1 @f$, the L1 norm; L2 norm, as in L2-SVM,
+ * is also available), and @f$
+ * \delta\{\mathrm{condition}\} = \left\{
+ * \begin{array}{lr}
+ * 1 & \mbox{if condition} \\
+ * -1 & \mbox{otherwise}
+ * \end{array} \right.
+ * @f$
+ *
+ * In an SVM, @f$ t \in \mathcal{R}^{N \times K} @f$ is the result of taking
+ * the inner product @f$ X^T W @f$ of the features
+ * @f$ X \in \mathcal{R}^{D \times N} @f$
+ * and the learned hyperplane parameters
+ * @f$ W \in \mathcal{R}^{D \times K} @f$. So, a Net with just an
+ * InnerProductLayer (with num_output = @f$k at f$) providing predictions to a
+ * HingeLossLayer is equivalent to an SVM (assuming it has no other learned
+ * outside the InnerProductLayer and no other losses outside the
+ * HingeLossLayer).
+ */
+template <typename Dtype>
+class HingeLossLayer : public LossLayer<Dtype> {
+ public:
+ explicit HingeLossLayer(const LayerParameter& param)
+ : LossLayer<Dtype>(param) {}
+
+ virtual inline const char* type() const { return "HingeLoss"; }
+
+ protected:
+ /// @copydoc HingeLossLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the hinge loss error gradient w.r.t. the predictions.
+ *
+ * Gradients cannot be computed with respect to the label inputs (bottom[1]),
+ * so this method ignores bottom[1] and requires !propagate_down[1], crashing
+ * if propagate_down[1] is set.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
+ * as @f$ \lambda @f$ is the coefficient of this layer's output
+ * @f$\ell_i at f$ in the overall Net loss
+ * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
+ * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
+ * (*Assuming that this top Blob is not used as a bottom (input) by any
+ * other layer of the Net.)
+ * @param propagate_down see Layer::Backward.
+ * propagate_down[1] must be false as we can't compute gradients with
+ * respect to the labels.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$t at f$; Backward computes diff
+ * @f$ \frac{\partial E}{\partial t} @f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels -- ignored as we can't compute their error gradients
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+
+} // namespace caffe
+
+#endif // CAFFE_HINGE_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/im2col_layer.hpp b/include/caffe/layers/im2col_layer.hpp
new file mode 100644
index 0000000..1d3b2eb
--- /dev/null
+++ b/include/caffe/layers/im2col_layer.hpp
@@ -0,0 +1,63 @@
+#ifndef CAFFE_IM2COL_LAYER_HPP_
+#define CAFFE_IM2COL_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief A helper for image operations that rearranges image regions into
+ * column vectors. Used by ConvolutionLayer to perform convolution
+ * by matrix multiplication.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class Im2colLayer : public Layer<Dtype> {
+ public:
+ explicit Im2colLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Im2col"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ /// @brief The spatial dimensions of a filter kernel.
+ Blob<int> kernel_shape_;
+ /// @brief The spatial dimensions of the stride.
+ Blob<int> stride_;
+ /// @brief The spatial dimensions of the padding.
+ Blob<int> pad_;
+
+ int num_spatial_axes_;
+ int bottom_dim_;
+ int top_dim_;
+
+ int channel_axis_;
+ int num_;
+ int channels_;
+
+ bool force_nd_im2col_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_IM2COL_LAYER_HPP_
diff --git a/include/caffe/layers/image_data_layer.hpp b/include/caffe/layers/image_data_layer.hpp
new file mode 100644
index 0000000..a0d3384
--- /dev/null
+++ b/include/caffe/layers/image_data_layer.hpp
@@ -0,0 +1,47 @@
+#ifndef CAFFE_IMAGE_DATA_LAYER_HPP_
+#define CAFFE_IMAGE_DATA_LAYER_HPP_
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/data_transformer.hpp"
+#include "caffe/internal_thread.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/layers/base_data_layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Provides data to the Net from image files.
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class ImageDataLayer : public BasePrefetchingDataLayer<Dtype> {
+ public:
+ explicit ImageDataLayer(const LayerParameter& param)
+ : BasePrefetchingDataLayer<Dtype>(param) {}
+ virtual ~ImageDataLayer();
+ virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "ImageData"; }
+ virtual inline int ExactNumBottomBlobs() const { return 0; }
+ virtual inline int ExactNumTopBlobs() const { return 2; }
+
+ protected:
+ shared_ptr<Caffe::RNG> prefetch_rng_;
+ virtual void ShuffleImages();
+ virtual void load_batch(Batch<Dtype>* batch);
+
+ vector<std::pair<std::string, int> > lines_;
+ int lines_id_;
+};
+
+
+} // namespace caffe
+
+#endif // CAFFE_IMAGE_DATA_LAYER_HPP_
diff --git a/include/caffe/layers/infogain_loss_layer.hpp b/include/caffe/layers/infogain_loss_layer.hpp
new file mode 100644
index 0000000..633f339
--- /dev/null
+++ b/include/caffe/layers/infogain_loss_layer.hpp
@@ -0,0 +1,110 @@
+#ifndef CAFFE_INFOGAIN_LOSS_LAYER_HPP_
+#define CAFFE_INFOGAIN_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief A generalization of MultinomialLogisticLossLayer that takes an
+ * "information gain" (infogain) matrix specifying the "value" of all label
+ * pairs.
+ *
+ * Equivalent to the MultinomialLogisticLossLayer if the infogain matrix is the
+ * identity.
+ *
+ * @param bottom input Blob vector (length 2-3)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ \hat{p} @f$, a Blob with values in
+ * @f$ [0, 1] @f$ indicating the predicted probability of each of the
+ * @f$ K = CHW @f$ classes. Each prediction vector @f$ \hat{p}_n @f$
+ * should sum to 1 as in a probability distribution: @f$
+ * \forall n \sum\limits_{k=1}^K \hat{p}_{nk} = 1 @f$.
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels @f$ l @f$, an integer-valued Blob with values
+ * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
+ * indicating the correct class label among the @f$ K @f$ classes
+ * -# @f$ (1 \times 1 \times K \times K) @f$
+ * (\b optional) the infogain matrix @f$ H @f$. This must be provided as
+ * the third bottom blob input if not provided as the infogain_mat in the
+ * InfogainLossParameter. If @f$ H = I @f$, this layer is equivalent to the
+ * MultinomialLogisticLossLayer.
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed infogain multinomial logistic loss: @f$ E =
+ * \frac{-1}{N} \sum\limits_{n=1}^N H_{l_n} \log(\hat{p}_n) =
+ * \frac{-1}{N} \sum\limits_{n=1}^N \sum\limits_{k=1}^{K} H_{l_n,k}
+ * \log(\hat{p}_{n,k})
+ * @f$, where @f$ H_{l_n} @f$ denotes row @f$l_n at f$ of @f$H at f$.
+ */
+template <typename Dtype>
+class InfogainLossLayer : public LossLayer<Dtype> {
+ public:
+ explicit InfogainLossLayer(const LayerParameter& param)
+ : LossLayer<Dtype>(param), infogain_() {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ // InfogainLossLayer takes 2-3 bottom Blobs; if there are 3 the third should
+ // be the infogain matrix. (Otherwise the infogain matrix is loaded from a
+ // file specified by LayerParameter.)
+ virtual inline int ExactNumBottomBlobs() const { return -1; }
+ virtual inline int MinBottomBlobs() const { return 2; }
+ virtual inline int MaxBottomBlobs() const { return 3; }
+
+ virtual inline const char* type() const { return "InfogainLoss"; }
+
+ protected:
+ /// @copydoc InfogainLossLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the infogain loss error gradient w.r.t. the predictions.
+ *
+ * Gradients cannot be computed with respect to the label inputs (bottom[1]),
+ * so this method ignores bottom[1] and requires !propagate_down[1], crashing
+ * if propagate_down[1] is set. (The same applies to the infogain matrix, if
+ * provided as bottom[2] rather than in the layer_param.)
+ *
+ * @param top output Blob vector (length 1), providing the error gradient
+ * with respect to the outputs
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
+ * as @f$ \lambda @f$ is the coefficient of this layer's output
+ * @f$\ell_i at f$ in the overall Net loss
+ * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
+ * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
+ * (*Assuming that this top Blob is not used as a bottom (input) by any
+ * other layer of the Net.)
+ * @param propagate_down see Layer::Backward.
+ * propagate_down[1] must be false as we can't compute gradients with
+ * respect to the labels (similarly for propagate_down[2] and the
+ * infogain matrix, if provided as bottom[2])
+ * @param bottom input Blob vector (length 2-3)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ \hat{p} @f$; Backward computes diff
+ * @f$ \frac{\partial E}{\partial \hat{p}} @f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels -- ignored as we can't compute their error gradients
+ * -# @f$ (1 \times 1 \times K \times K) @f$
+ * (\b optional) the information gain matrix -- ignored as its error
+ * gradient computation is not implemented.
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ Blob<Dtype> infogain_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_INFOGAIN_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/inner_product_layer.hpp b/include/caffe/layers/inner_product_layer.hpp
new file mode 100644
index 0000000..250576a
--- /dev/null
+++ b/include/caffe/layers/inner_product_layer.hpp
@@ -0,0 +1,51 @@
+#ifndef CAFFE_INNER_PRODUCT_LAYER_HPP_
+#define CAFFE_INNER_PRODUCT_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Also known as a "fully-connected" layer, computes an inner product
+ * with a set of learned weights, and (optionally) adds biases.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class InnerProductLayer : public Layer<Dtype> {
+ public:
+ explicit InnerProductLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "InnerProduct"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int M_;
+ int K_;
+ int N_;
+ bool bias_term_;
+ Blob<Dtype> bias_multiplier_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_INNER_PRODUCT_LAYER_HPP_
diff --git a/include/caffe/layers/log_layer.hpp b/include/caffe/layers/log_layer.hpp
new file mode 100644
index 0000000..7d037d2
--- /dev/null
+++ b/include/caffe/layers/log_layer.hpp
@@ -0,0 +1,82 @@
+#ifndef CAFFE_LOG_LAYER_HPP_
+#define CAFFE_LOG_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes @f$ y = log_{\gamma}(\alpha x + \beta) @f$,
+ * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$,
+ * and base @f$ \gamma @f$.
+ */
+template <typename Dtype>
+class LogLayer : public NeuronLayer<Dtype> {
+ public:
+ /**
+ * @param param provides LogParameter log_param,
+ * with LogLayer options:
+ * - scale (\b optional, default 1) the scale @f$ \alpha @f$
+ * - shift (\b optional, default 0) the shift @f$ \beta @f$
+ * - base (\b optional, default -1 for a value of @f$ e \approx 2.718 @f$)
+ * the base @f$ \gamma @f$
+ */
+ explicit LogLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Log"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = log_{\gamma}(\alpha x + \beta)
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the exp inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x} =
+ * \frac{\partial E}{\partial y} y \alpha \log_e(gamma)
+ * @f$ if propagate_down[0]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ Dtype base_scale_;
+ Dtype input_scale_, input_shift_;
+ Dtype backward_num_scale_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_LOG_LAYER_HPP_
diff --git a/include/caffe/layers/loss_layer.hpp b/include/caffe/layers/loss_layer.hpp
new file mode 100644
index 0000000..dbdf612
--- /dev/null
+++ b/include/caffe/layers/loss_layer.hpp
@@ -0,0 +1,53 @@
+#ifndef CAFFE_LOSS_LAYER_HPP_
+#define CAFFE_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+const float kLOG_THRESHOLD = 1e-20;
+
+/**
+ * @brief An interface for Layer%s that take two Blob%s as input -- usually
+ * (1) predictions and (2) ground-truth labels -- and output a
+ * singleton Blob representing the loss.
+ *
+ * LossLayers are typically only capable of backpropagating to their first input
+ * -- the predictions.
+ */
+template <typename Dtype>
+class LossLayer : public Layer<Dtype> {
+ public:
+ explicit LossLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(
+ const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(
+ const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top);
+
+ virtual inline int ExactNumBottomBlobs() const { return 2; }
+
+ /**
+ * @brief For convenience and backwards compatibility, instruct the Net to
+ * automatically allocate a single top Blob for LossLayers, into which
+ * they output their singleton loss, (even if the user didn't specify
+ * one in the prototxt, etc.).
+ */
+ virtual inline bool AutoTopBlobs() const { return true; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+ /**
+ * We usually cannot backpropagate to the labels; ignore force_backward for
+ * these inputs.
+ */
+ virtual inline bool AllowForceBackward(const int bottom_index) const {
+ return bottom_index != 1;
+ }
+};
+
+} // namespace caffe
+
+#endif // CAFFE_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/lrn_layer.hpp b/include/caffe/layers/lrn_layer.hpp
new file mode 100644
index 0000000..06cf71a
--- /dev/null
+++ b/include/caffe/layers/lrn_layer.hpp
@@ -0,0 +1,94 @@
+#ifndef CAFFE_LRN_LAYER_HPP_
+#define CAFFE_LRN_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/eltwise_layer.hpp"
+#include "caffe/layers/pooling_layer.hpp"
+#include "caffe/layers/power_layer.hpp"
+#include "caffe/layers/split_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Normalize the input in a local region across or within feature maps.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class LRNLayer : public Layer<Dtype> {
+ public:
+ explicit LRNLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "LRN"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ virtual void CrossChannelForward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void CrossChannelForward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void WithinChannelForward(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void CrossChannelBackward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void CrossChannelBackward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void WithinChannelBackward(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int size_;
+ int pre_pad_;
+ Dtype alpha_;
+ Dtype beta_;
+ Dtype k_;
+ int num_;
+ int channels_;
+ int height_;
+ int width_;
+
+ // Fields used for normalization ACROSS_CHANNELS
+ // scale_ stores the intermediate summing results
+ Blob<Dtype> scale_;
+
+ // Fields used for normalization WITHIN_CHANNEL
+ shared_ptr<SplitLayer<Dtype> > split_layer_;
+ vector<Blob<Dtype>*> split_top_vec_;
+ shared_ptr<PowerLayer<Dtype> > square_layer_;
+ Blob<Dtype> square_input_;
+ Blob<Dtype> square_output_;
+ vector<Blob<Dtype>*> square_bottom_vec_;
+ vector<Blob<Dtype>*> square_top_vec_;
+ shared_ptr<PoolingLayer<Dtype> > pool_layer_;
+ Blob<Dtype> pool_output_;
+ vector<Blob<Dtype>*> pool_top_vec_;
+ shared_ptr<PowerLayer<Dtype> > power_layer_;
+ Blob<Dtype> power_output_;
+ vector<Blob<Dtype>*> power_top_vec_;
+ shared_ptr<EltwiseLayer<Dtype> > product_layer_;
+ Blob<Dtype> product_input_;
+ vector<Blob<Dtype>*> product_bottom_vec_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_LRN_LAYER_HPP_
diff --git a/include/caffe/layers/memory_data_layer.hpp b/include/caffe/layers/memory_data_layer.hpp
new file mode 100644
index 0000000..8abcc8c
--- /dev/null
+++ b/include/caffe/layers/memory_data_layer.hpp
@@ -0,0 +1,63 @@
+#ifndef CAFFE_MEMORY_DATA_LAYER_HPP_
+#define CAFFE_MEMORY_DATA_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/base_data_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Provides data to the Net from memory.
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class MemoryDataLayer : public BaseDataLayer<Dtype> {
+ public:
+ explicit MemoryDataLayer(const LayerParameter& param)
+ : BaseDataLayer<Dtype>(param), has_new_data_(false) {}
+ virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "MemoryData"; }
+ virtual inline int ExactNumBottomBlobs() const { return 0; }
+ virtual inline int ExactNumTopBlobs() const { return 2; }
+
+ virtual void AddDatumVector(const vector<Datum>& datum_vector);
+#ifdef USE_OPENCV
+ virtual void AddMatVector(const vector<cv::Mat>& mat_vector,
+ const vector<int>& labels);
+#endif // USE_OPENCV
+
+ // Reset should accept const pointers, but can't, because the memory
+ // will be given to Blob, which is mutable
+ void Reset(Dtype* data, Dtype* label, int n);
+ void set_batch_size(int new_size);
+
+ int batch_size() { return batch_size_; }
+ int channels() { return channels_; }
+ int height() { return height_; }
+ int width() { return width_; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ int batch_size_, channels_, height_, width_, size_;
+ Dtype* data_;
+ Dtype* labels_;
+ int n_;
+ size_t pos_;
+ Blob<Dtype> added_data_;
+ Blob<Dtype> added_label_;
+ bool has_new_data_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_MEMORY_DATA_LAYER_HPP_
diff --git a/include/caffe/layers/multinomial_logistic_loss_layer.hpp b/include/caffe/layers/multinomial_logistic_loss_layer.hpp
new file mode 100644
index 0000000..3977cf9
--- /dev/null
+++ b/include/caffe/layers/multinomial_logistic_loss_layer.hpp
@@ -0,0 +1,92 @@
+#ifndef CAFFE_MULTINOMIAL_LOGISTIC_LOSS_LAYER_HPP_
+#define CAFFE_MULTINOMIAL_LOGISTIC_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes the multinomial logistic loss for a one-of-many
+ * classification task, directly taking a predicted probability
+ * distribution as input.
+ *
+ * When predictions are not already a probability distribution, you should
+ * instead use the SoftmaxWithLossLayer, which maps predictions to a
+ * distribution using the SoftmaxLayer, before computing the multinomial
+ * logistic loss. The SoftmaxWithLossLayer should be preferred over separate
+ * SoftmaxLayer + MultinomialLogisticLossLayer
+ * as its gradient computation is more numerically stable.
+ *
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ \hat{p} @f$, a Blob with values in
+ * @f$ [0, 1] @f$ indicating the predicted probability of each of the
+ * @f$ K = CHW @f$ classes. Each prediction vector @f$ \hat{p}_n @f$
+ * should sum to 1 as in a probability distribution: @f$
+ * \forall n \sum\limits_{k=1}^K \hat{p}_{nk} = 1 @f$.
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels @f$ l @f$, an integer-valued Blob with values
+ * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
+ * indicating the correct class label among the @f$ K @f$ classes
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed multinomial logistic loss: @f$ E =
+ * \frac{-1}{N} \sum\limits_{n=1}^N \log(\hat{p}_{n,l_n})
+ * @f$
+ */
+template <typename Dtype>
+class MultinomialLogisticLossLayer : public LossLayer<Dtype> {
+ public:
+ explicit MultinomialLogisticLossLayer(const LayerParameter& param)
+ : LossLayer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "MultinomialLogisticLoss"; }
+
+ protected:
+ /// @copydoc MultinomialLogisticLossLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the multinomial logistic loss error gradient w.r.t. the
+ * predictions.
+ *
+ * Gradients cannot be computed with respect to the label inputs (bottom[1]),
+ * so this method ignores bottom[1] and requires !propagate_down[1], crashing
+ * if propagate_down[1] is set.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
+ * as @f$ \lambda @f$ is the coefficient of this layer's output
+ * @f$\ell_i at f$ in the overall Net loss
+ * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
+ * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
+ * (*Assuming that this top Blob is not used as a bottom (input) by any
+ * other layer of the Net.)
+ * @param propagate_down see Layer::Backward.
+ * propagate_down[1] must be false as we can't compute gradients with
+ * respect to the labels.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ \hat{p} @f$; Backward computes diff
+ * @f$ \frac{\partial E}{\partial \hat{p}} @f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels -- ignored as we can't compute their error gradients
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_MULTINOMIAL_LOGISTIC_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/mvn_layer.hpp b/include/caffe/layers/mvn_layer.hpp
new file mode 100644
index 0000000..3a235ce
--- /dev/null
+++ b/include/caffe/layers/mvn_layer.hpp
@@ -0,0 +1,48 @@
+#ifndef CAFFE_MVN_LAYER_HPP_
+#define CAFFE_MVN_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Normalizes the input to have 0-mean and/or unit (1) variance.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class MVNLayer : public Layer<Dtype> {
+ public:
+ explicit MVNLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "MVN"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ Blob<Dtype> mean_, variance_, temp_;
+
+ /// sum_multiplier is used to carry out sum using BLAS
+ Blob<Dtype> sum_multiplier_;
+ Dtype eps_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_MVN_LAYER_HPP_
diff --git a/include/caffe/layers/neuron_layer.hpp b/include/caffe/layers/neuron_layer.hpp
new file mode 100644
index 0000000..10c108c
--- /dev/null
+++ b/include/caffe/layers/neuron_layer.hpp
@@ -0,0 +1,32 @@
+#ifndef CAFFE_NEURON_LAYER_HPP_
+#define CAFFE_NEURON_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief An interface for layers that take one blob as input (@f$ x @f$)
+ * and produce one equally-sized blob as output (@f$ y @f$), where
+ * each element of the output depends only on the corresponding input
+ * element.
+ */
+template <typename Dtype>
+class NeuronLayer : public Layer<Dtype> {
+ public:
+ explicit NeuronLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+};
+
+} // namespace caffe
+
+#endif // CAFFE_NEURON_LAYER_HPP_
diff --git a/include/caffe/layers/pooling_layer.hpp b/include/caffe/layers/pooling_layer.hpp
new file mode 100644
index 0000000..f4d6803
--- /dev/null
+++ b/include/caffe/layers/pooling_layer.hpp
@@ -0,0 +1,60 @@
+#ifndef CAFFE_POOLING_LAYER_HPP_
+#define CAFFE_POOLING_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Pools the input image by taking the max, average, etc. within regions.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class PoolingLayer : public Layer<Dtype> {
+ public:
+ explicit PoolingLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Pooling"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int MinTopBlobs() const { return 1; }
+ // MAX POOL layers can output an extra top blob for the mask;
+ // others can only output the pooled inputs.
+ virtual inline int MaxTopBlobs() const {
+ return (this->layer_param_.pooling_param().pool() ==
+ PoolingParameter_PoolMethod_MAX) ? 2 : 1;
+ }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int kernel_h_, kernel_w_;
+ int stride_h_, stride_w_;
+ int pad_h_, pad_w_;
+ int channels_;
+ int height_, width_;
+ int pooled_height_, pooled_width_;
+ bool global_pooling_;
+ Blob<Dtype> rand_idx_;
+ Blob<int> max_idx_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_POOLING_LAYER_HPP_
diff --git a/include/caffe/layers/power_layer.hpp b/include/caffe/layers/power_layer.hpp
new file mode 100644
index 0000000..6ecbafc
--- /dev/null
+++ b/include/caffe/layers/power_layer.hpp
@@ -0,0 +1,89 @@
+#ifndef CAFFE_POWER_LAYER_HPP_
+#define CAFFE_POWER_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes @f$ y = (\alpha x + \beta) ^ \gamma @f$,
+ * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$,
+ * and power @f$ \gamma @f$.
+ */
+template <typename Dtype>
+class PowerLayer : public NeuronLayer<Dtype> {
+ public:
+ /**
+ * @param param provides PowerParameter power_param,
+ * with PowerLayer options:
+ * - scale (\b optional, default 1) the scale @f$ \alpha @f$
+ * - shift (\b optional, default 0) the shift @f$ \beta @f$
+ * - power (\b optional, default 1) the power @f$ \gamma @f$
+ */
+ explicit PowerLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Power"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = (\alpha x + \beta) ^ \gamma
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the power inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x} =
+ * \frac{\partial E}{\partial y}
+ * \alpha \gamma (\alpha x + \beta) ^ {\gamma - 1} =
+ * \frac{\partial E}{\partial y}
+ * \frac{\alpha \gamma y}{\alpha x + \beta}
+ * @f$ if propagate_down[0]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ /// @brief @f$ \gamma @f$ from layer_param_.power_param()
+ Dtype power_;
+ /// @brief @f$ \alpha @f$ from layer_param_.power_param()
+ Dtype scale_;
+ /// @brief @f$ \beta @f$ from layer_param_.power_param()
+ Dtype shift_;
+ /// @brief Result of @f$ \alpha \gamma @f$
+ Dtype diff_scale_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_POWER_LAYER_HPP_
diff --git a/include/caffe/layers/prelu_layer.hpp b/include/caffe/layers/prelu_layer.hpp
new file mode 100644
index 0000000..3ddfb48
--- /dev/null
+++ b/include/caffe/layers/prelu_layer.hpp
@@ -0,0 +1,101 @@
+#ifndef CAFFE_PRELU_LAYER_HPP_
+#define CAFFE_PRELU_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Parameterized Rectified Linear Unit non-linearity @f$
+ * y_i = \max(0, x_i) + a_i \min(0, x_i)
+ * @f$. The differences from ReLULayer are 1) negative slopes are
+ * learnable though backprop and 2) negative slopes can vary across
+ * channels. The number of axes of input blob should be greater than or
+ * equal to 2. The 1st axis (0-based) is seen as channels.
+ */
+template <typename Dtype>
+class PReLULayer : public NeuronLayer<Dtype> {
+ public:
+ /**
+ * @param param provides PReLUParameter prelu_param,
+ * with PReLULayer options:
+ * - filler (\b optional, FillerParameter,
+ * default {'type': constant 'value':0.25}).
+ * - channel_shared (\b optional, default false).
+ * negative slopes are shared across channels.
+ */
+ explicit PReLULayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "PReLU"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times ...) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times ...) @f$
+ * the computed outputs for each channel @f$i at f$ @f$
+ * y_i = \max(0, x_i) + a_i \min(0, x_i)
+ * @f$.
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the PReLU inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times ...) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times ...) @f$
+ * the inputs @f$ x @f$; For each channel @f$i at f$, backward fills their
+ * diff with gradients @f$
+ * \frac{\partial E}{\partial x_i} = \left\{
+ * \begin{array}{lr}
+ * a_i \frac{\partial E}{\partial y_i} & \mathrm{if} \; x_i \le 0 \\
+ * \frac{\partial E}{\partial y_i} & \mathrm{if} \; x_i > 0
+ * \end{array} \right.
+ * @f$.
+ * If param_propagate_down_[0] is true, it fills the diff with gradients
+ * @f$
+ * \frac{\partial E}{\partial a_i} = \left\{
+ * \begin{array}{lr}
+ * \sum_{x_i} x_i \frac{\partial E}{\partial y_i} & \mathrm{if} \; x_i \le 0 \\
+ * 0 & \mathrm{if} \; x_i > 0
+ * \end{array} \right.
+ * @f$.
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ bool channel_shared_;
+ Blob<Dtype> multiplier_; // dot multiplier for backward computation of params
+ Blob<Dtype> backward_buff_; // temporary buffer for backward computation
+ Blob<Dtype> bottom_memory_; // memory for in-place computation
+};
+
+} // namespace caffe
+
+#endif // CAFFE_PRELU_LAYER_HPP_
diff --git a/include/caffe/python_layer.hpp b/include/caffe/layers/python_layer.hpp
similarity index 82%
rename from include/caffe/python_layer.hpp
rename to include/caffe/layers/python_layer.hpp
index c43c1e8..b839d52 100644
--- a/include/caffe/python_layer.hpp
+++ b/include/caffe/layers/python_layer.hpp
@@ -18,6 +18,12 @@ class PythonLayer : public Layer<Dtype> {
virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
+ // Disallow PythonLayer in MultiGPU training stage, due to GIL issues
+ // Details: https://github.com/BVLC/caffe/issues/2936
+ if (this->phase_ == TRAIN && Caffe::solver_count() > 1
+ && !ShareInParallel()) {
+ LOG(FATAL) << "PythonLayer is not implemented in Multi-GPU training";
+ }
self_.attr("param_str") = bp::str(
this->layer_param_.python_param().param_str());
self_.attr("setup")(bottom, top);
diff --git a/include/caffe/layers/reduction_layer.hpp b/include/caffe/layers/reduction_layer.hpp
new file mode 100644
index 0000000..804a495
--- /dev/null
+++ b/include/caffe/layers/reduction_layer.hpp
@@ -0,0 +1,59 @@
+#ifndef CAFFE_REDUCTION_LAYER_HPP_
+#define CAFFE_REDUCTION_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Compute "reductions" -- operations that return a scalar output Blob
+ * for an input Blob of arbitrary size, such as the sum, absolute sum,
+ * and sum of squares.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class ReductionLayer : public Layer<Dtype> {
+ public:
+ explicit ReductionLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Reduction"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ /// @brief the reduction operation performed by the layer
+ ReductionParameter_ReductionOp op_;
+ /// @brief a scalar coefficient applied to all outputs
+ Dtype coeff_;
+ /// @brief the index of the first input axis to reduce
+ int axis_;
+ /// @brief the number of reductions performed
+ int num_;
+ /// @brief the input size of each reduction
+ int dim_;
+ /// @brief a helper Blob used for summation (op_ == SUM)
+ Blob<Dtype> sum_multiplier_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_REDUCTION_LAYER_HPP_
diff --git a/include/caffe/layers/relu_layer.hpp b/include/caffe/layers/relu_layer.hpp
new file mode 100644
index 0000000..d7a73f7
--- /dev/null
+++ b/include/caffe/layers/relu_layer.hpp
@@ -0,0 +1,85 @@
+#ifndef CAFFE_RELU_LAYER_HPP_
+#define CAFFE_RELU_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Rectified Linear Unit non-linearity @f$ y = \max(0, x) @f$.
+ * The simple max is fast to compute, and the function does not saturate.
+ */
+template <typename Dtype>
+class ReLULayer : public NeuronLayer<Dtype> {
+ public:
+ /**
+ * @param param provides ReLUParameter relu_param,
+ * with ReLULayer options:
+ * - negative_slope (\b optional, default 0).
+ * the value @f$ \nu @f$ by which negative values are multiplied.
+ */
+ explicit ReLULayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+
+ virtual inline const char* type() const { return "ReLU"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = \max(0, x)
+ * @f$ by default. If a non-zero negative_slope @f$ \nu @f$ is provided,
+ * the computed outputs are @f$ y = \max(0, x) + \nu \min(0, x) @f$.
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the ReLU inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x} = \left\{
+ * \begin{array}{lr}
+ * 0 & \mathrm{if} \; x \le 0 \\
+ * \frac{\partial E}{\partial y} & \mathrm{if} \; x > 0
+ * \end{array} \right.
+ * @f$ if propagate_down[0], by default.
+ * If a non-zero negative_slope @f$ \nu @f$ is provided,
+ * the computed gradients are @f$
+ * \frac{\partial E}{\partial x} = \left\{
+ * \begin{array}{lr}
+ * \nu \frac{\partial E}{\partial y} & \mathrm{if} \; x \le 0 \\
+ * \frac{\partial E}{\partial y} & \mathrm{if} \; x > 0
+ * \end{array} \right.
+ * @f$.
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_RELU_LAYER_HPP_
diff --git a/include/caffe/layers/reshape_layer.hpp b/include/caffe/layers/reshape_layer.hpp
new file mode 100644
index 0000000..d11e063
--- /dev/null
+++ b/include/caffe/layers/reshape_layer.hpp
@@ -0,0 +1,52 @@
+#ifndef CAFFE_XXX_LAYER_HPP_
+#define CAFFE_XXX_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/*
+ * @brief Reshapes the input Blob into an arbitrary-sized output Blob.
+ *
+ * Note: similarly to FlattenLayer, this layer does not change the input values
+ * (see FlattenLayer, Blob::ShareData and Blob::ShareDiff).
+ */
+template <typename Dtype>
+class ReshapeLayer : public Layer<Dtype> {
+ public:
+ explicit ReshapeLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Reshape"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {}
+
+ /// @brief vector of axes indices whose dimensions we'll copy from the bottom
+ vector<int> copy_axes_;
+ /// @brief the index of the axis whose dimension we infer, or -1 if none
+ int inferred_axis_;
+ /// @brief the product of the "constant" output dimensions
+ int constant_count_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_XXX_LAYER_HPP_
diff --git a/include/caffe/layers/sigmoid_cross_entropy_loss_layer.hpp b/include/caffe/layers/sigmoid_cross_entropy_loss_layer.hpp
new file mode 100644
index 0000000..598dca5
--- /dev/null
+++ b/include/caffe/layers/sigmoid_cross_entropy_loss_layer.hpp
@@ -0,0 +1,110 @@
+#ifndef CAFFE_SIGMOID_CROSS_ENTROPY_LOSS_LAYER_HPP_
+#define CAFFE_SIGMOID_CROSS_ENTROPY_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+#include "caffe/layers/sigmoid_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes the cross-entropy (logistic) loss @f$
+ * E = \frac{-1}{n} \sum\limits_{n=1}^N \left[
+ * p_n \log \hat{p}_n +
+ * (1 - p_n) \log(1 - \hat{p}_n)
+ * \right]
+ * @f$, often used for predicting targets interpreted as probabilities.
+ *
+ * This layer is implemented rather than separate
+ * SigmoidLayer + CrossEntropyLayer
+ * as its gradient computation is more numerically stable.
+ * At test time, this layer can be replaced simply by a SigmoidLayer.
+ *
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the scores @f$ x \in [-\infty, +\infty]@f$,
+ * which this layer maps to probability predictions
+ * @f$ \hat{p}_n = \sigma(x_n) \in [0, 1] @f$
+ * using the sigmoid function @f$ \sigma(.) @f$ (see SigmoidLayer).
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the targets @f$ y \in [0, 1] @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed cross-entropy loss: @f$
+ * E = \frac{-1}{n} \sum\limits_{n=1}^N \left[
+ * p_n \log \hat{p}_n + (1 - p_n) \log(1 - \hat{p}_n)
+ * \right]
+ * @f$
+ */
+template <typename Dtype>
+class SigmoidCrossEntropyLossLayer : public LossLayer<Dtype> {
+ public:
+ explicit SigmoidCrossEntropyLossLayer(const LayerParameter& param)
+ : LossLayer<Dtype>(param),
+ sigmoid_layer_(new SigmoidLayer<Dtype>(param)),
+ sigmoid_output_(new Blob<Dtype>()) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "SigmoidCrossEntropyLoss"; }
+
+ protected:
+ /// @copydoc SigmoidCrossEntropyLossLayer
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the sigmoid cross-entropy loss error gradient w.r.t. the
+ * predictions.
+ *
+ * Gradients cannot be computed with respect to the target inputs (bottom[1]),
+ * so this method ignores bottom[1] and requires !propagate_down[1], crashing
+ * if propagate_down[1] is set.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
+ * as @f$ \lambda @f$ is the coefficient of this layer's output
+ * @f$\ell_i at f$ in the overall Net loss
+ * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
+ * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
+ * (*Assuming that this top Blob is not used as a bottom (input) by any
+ * other layer of the Net.)
+ * @param propagate_down see Layer::Backward.
+ * propagate_down[1] must be false as gradient computation with respect
+ * to the targets is not implemented.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$x at f$; Backward computes diff
+ * @f$ \frac{\partial E}{\partial x} =
+ * \frac{1}{n} \sum\limits_{n=1}^N (\hat{p}_n - p_n)
+ * @f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels -- ignored as we can't compute their error gradients
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ /// The internal SigmoidLayer used to map predictions to probabilities.
+ shared_ptr<SigmoidLayer<Dtype> > sigmoid_layer_;
+ /// sigmoid_output stores the output of the SigmoidLayer.
+ shared_ptr<Blob<Dtype> > sigmoid_output_;
+ /// bottom vector holder to call the underlying SigmoidLayer::Forward
+ vector<Blob<Dtype>*> sigmoid_bottom_vec_;
+ /// top vector holder to call the underlying SigmoidLayer::Forward
+ vector<Blob<Dtype>*> sigmoid_top_vec_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SIGMOID_CROSS_ENTROPY_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/sigmoid_layer.hpp b/include/caffe/layers/sigmoid_layer.hpp
new file mode 100644
index 0000000..ac0f692
--- /dev/null
+++ b/include/caffe/layers/sigmoid_layer.hpp
@@ -0,0 +1,71 @@
+#ifndef CAFFE_SIGMOID_LAYER_HPP_
+#define CAFFE_SIGMOID_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Sigmoid function non-linearity @f$
+ * y = (1 + \exp(-x))^{-1}
+ * @f$, a classic choice in neural networks.
+ *
+ * Note that the gradient vanishes as the values move away from 0.
+ * The ReLULayer is often a better choice for this reason.
+ */
+template <typename Dtype>
+class SigmoidLayer : public NeuronLayer<Dtype> {
+ public:
+ explicit SigmoidLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+
+ virtual inline const char* type() const { return "Sigmoid"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = (1 + \exp(-x))^{-1}
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the sigmoid inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x}
+ * = \frac{\partial E}{\partial y} y (1 - y)
+ * @f$ if propagate_down[0]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SIGMOID_LAYER_HPP_
diff --git a/include/caffe/layers/silence_layer.hpp b/include/caffe/layers/silence_layer.hpp
new file mode 100644
index 0000000..fba087f
--- /dev/null
+++ b/include/caffe/layers/silence_layer.hpp
@@ -0,0 +1,43 @@
+#ifndef CAFFE_SILENCE_LAYER_HPP_
+#define CAFFE_SILENCE_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Ignores bottom blobs while producing no top blobs. (This is useful
+ * to suppress outputs during testing.)
+ */
+template <typename Dtype>
+class SilenceLayer : public Layer<Dtype> {
+ public:
+ explicit SilenceLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+
+ virtual inline const char* type() const { return "Silence"; }
+ virtual inline int MinBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 0; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {}
+ // We can't define Forward_gpu here, since STUB_GPU will provide
+ // its own definition for CPU_ONLY mode.
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SILENCE_LAYER_HPP_
diff --git a/include/caffe/layers/slice_layer.hpp b/include/caffe/layers/slice_layer.hpp
new file mode 100644
index 0000000..10a0abb
--- /dev/null
+++ b/include/caffe/layers/slice_layer.hpp
@@ -0,0 +1,51 @@
+#ifndef CAFFE_SLICE_LAYER_HPP_
+#define CAFFE_SLICE_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Takes a Blob and slices it along either the num or channel dimension,
+ * outputting multiple sliced Blob results.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class SliceLayer : public Layer<Dtype> {
+ public:
+ explicit SliceLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Slice"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int MinTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int count_;
+ int num_slices_;
+ int slice_size_;
+ int slice_axis_;
+ vector<int> slice_point_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SLICE_LAYER_HPP_
diff --git a/include/caffe/layers/softmax_layer.hpp b/include/caffe/layers/softmax_layer.hpp
new file mode 100644
index 0000000..c65b870
--- /dev/null
+++ b/include/caffe/layers/softmax_layer.hpp
@@ -0,0 +1,50 @@
+#ifndef CAFFE_SOFTMAX_LAYER_HPP_
+#define CAFFE_SOFTMAX_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Computes the softmax function.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class SoftmaxLayer : public Layer<Dtype> {
+ public:
+ explicit SoftmaxLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Softmax"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int outer_num_;
+ int inner_num_;
+ int softmax_axis_;
+ /// sum_multiplier is used to carry out sum using BLAS
+ Blob<Dtype> sum_multiplier_;
+ /// scale is an intermediate Blob to hold temporary results.
+ Blob<Dtype> scale_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SOFTMAX_LAYER_HPP_
diff --git a/include/caffe/layers/softmax_loss_layer.hpp b/include/caffe/layers/softmax_loss_layer.hpp
new file mode 100644
index 0000000..f07e8a0
--- /dev/null
+++ b/include/caffe/layers/softmax_loss_layer.hpp
@@ -0,0 +1,130 @@
+#ifndef CAFFE_SOFTMAX_WITH_LOSS_LAYER_HPP_
+#define CAFFE_SOFTMAX_WITH_LOSS_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/loss_layer.hpp"
+#include "caffe/layers/softmax_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Computes the multinomial logistic loss for a one-of-many
+ * classification task, passing real-valued predictions through a
+ * softmax to get a probability distribution over classes.
+ *
+ * This layer should be preferred over separate
+ * SoftmaxLayer + MultinomialLogisticLossLayer
+ * as its gradient computation is more numerically stable.
+ * At test time, this layer can be replaced simply by a SoftmaxLayer.
+ *
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ x @f$, a Blob with values in
+ * @f$ [-\infty, +\infty] @f$ indicating the predicted score for each of
+ * the @f$ K = CHW @f$ classes. This layer maps these scores to a
+ * probability distribution over classes using the softmax function
+ * @f$ \hat{p}_{nk} = \exp(x_{nk}) /
+ * \left[\sum_{k'} \exp(x_{nk'})\right] @f$ (see SoftmaxLayer).
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels @f$ l @f$, an integer-valued Blob with values
+ * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
+ * indicating the correct class label among the @f$ K @f$ classes
+ * @param top output Blob vector (length 1)
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * the computed cross-entropy classification loss: @f$ E =
+ * \frac{-1}{N} \sum\limits_{n=1}^N \log(\hat{p}_{n,l_n})
+ * @f$, for softmax output class probabilites @f$ \hat{p} @f$
+ */
+template <typename Dtype>
+class SoftmaxWithLossLayer : public LossLayer<Dtype> {
+ public:
+ /**
+ * @param param provides LossParameter loss_param, with options:
+ * - ignore_label (optional)
+ * Specify a label value that should be ignored when computing the loss.
+ * - normalize (optional, default true)
+ * If true, the loss is normalized by the number of (nonignored) labels
+ * present; otherwise the loss is simply summed over spatial locations.
+ */
+ explicit SoftmaxWithLossLayer(const LayerParameter& param)
+ : LossLayer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "SoftmaxWithLoss"; }
+ virtual inline int ExactNumTopBlobs() const { return -1; }
+ virtual inline int MinTopBlobs() const { return 1; }
+ virtual inline int MaxTopBlobs() const { return 2; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ /**
+ * @brief Computes the softmax loss error gradient w.r.t. the predictions.
+ *
+ * Gradients cannot be computed with respect to the label inputs (bottom[1]),
+ * so this method ignores bottom[1] and requires !propagate_down[1], crashing
+ * if propagate_down[1] is set.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (1 \times 1 \times 1 \times 1) @f$
+ * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
+ * as @f$ \lambda @f$ is the coefficient of this layer's output
+ * @f$\ell_i at f$ in the overall Net loss
+ * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
+ * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
+ * (*Assuming that this top Blob is not used as a bottom (input) by any
+ * other layer of the Net.)
+ * @param propagate_down see Layer::Backward.
+ * propagate_down[1] must be false as we can't compute gradients with
+ * respect to the labels.
+ * @param bottom input Blob vector (length 2)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the predictions @f$ x @f$; Backward computes diff
+ * @f$ \frac{\partial E}{\partial x} @f$
+ * -# @f$ (N \times 1 \times 1 \times 1) @f$
+ * the labels -- ignored as we can't compute their error gradients
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ /// Read the normalization mode parameter and compute the normalizer based
+ /// on the blob size. If normalization_mode is VALID, the count of valid
+ /// outputs will be read from valid_count, unless it is -1 in which case
+ /// all outputs are assumed to be valid.
+ virtual Dtype get_normalizer(
+ LossParameter_NormalizationMode normalization_mode, int valid_count);
+
+ /// The internal SoftmaxLayer used to map predictions to a distribution.
+ shared_ptr<Layer<Dtype> > softmax_layer_;
+ /// prob stores the output probability predictions from the SoftmaxLayer.
+ Blob<Dtype> prob_;
+ /// bottom vector holder used in call to the underlying SoftmaxLayer::Forward
+ vector<Blob<Dtype>*> softmax_bottom_vec_;
+ /// top vector holder used in call to the underlying SoftmaxLayer::Forward
+ vector<Blob<Dtype>*> softmax_top_vec_;
+ /// Whether to ignore instances with a certain label.
+ bool has_ignore_label_;
+ /// The label indicating that an instance should be ignored.
+ int ignore_label_;
+ /// How to normalize the output loss.
+ LossParameter_NormalizationMode normalization_;
+
+ int softmax_axis_, outer_num_, inner_num_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SOFTMAX_WITH_LOSS_LAYER_HPP_
diff --git a/include/caffe/layers/split_layer.hpp b/include/caffe/layers/split_layer.hpp
new file mode 100644
index 0000000..8140dfc
--- /dev/null
+++ b/include/caffe/layers/split_layer.hpp
@@ -0,0 +1,45 @@
+#ifndef CAFFE_SPLIT_LAYER_HPP_
+#define CAFFE_SPLIT_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Creates a "split" path in the network by copying the bottom Blob
+ * into multiple top Blob%s to be used by multiple consuming layers.
+ *
+ * TODO(dox): thorough documentation for Forward, Backward, and proto params.
+ */
+template <typename Dtype>
+class SplitLayer : public Layer<Dtype> {
+ public:
+ explicit SplitLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Split"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int MinTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ int count_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SPLIT_LAYER_HPP_
diff --git a/include/caffe/layers/spp_layer.hpp b/include/caffe/layers/spp_layer.hpp
new file mode 100644
index 0000000..9f145cc
--- /dev/null
+++ b/include/caffe/layers/spp_layer.hpp
@@ -0,0 +1,76 @@
+#ifndef CAFFE_SPP_LAYER_HPP_
+#define CAFFE_SPP_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Does spatial pyramid pooling on the input image
+ * by taking the max, average, etc. within regions
+ * so that the result vector of different sized
+ * images are of the same size.
+ */
+template <typename Dtype>
+class SPPLayer : public Layer<Dtype> {
+ public:
+ explicit SPPLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "SPP"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ // calculates the kernel and stride dimensions for the pooling layer,
+ // returns a correctly configured LayerParameter for a PoolingLayer
+ virtual LayerParameter GetPoolingParam(const int pyramid_level,
+ const int bottom_h, const int bottom_w, const SPPParameter spp_param);
+
+ int pyramid_height_;
+ int bottom_h_, bottom_w_;
+ int num_;
+ int channels_;
+ int kernel_h_, kernel_w_;
+ int pad_h_, pad_w_;
+ bool reshaped_first_time_;
+
+ /// the internal Split layer that feeds the pooling layers
+ shared_ptr<SplitLayer<Dtype> > split_layer_;
+ /// top vector holder used in call to the underlying SplitLayer::Forward
+ vector<Blob<Dtype>*> split_top_vec_;
+ /// bottom vector holder used in call to the underlying PoolingLayer::Forward
+ vector<vector<Blob<Dtype>*>*> pooling_bottom_vecs_;
+ /// the internal Pooling layers of different kernel sizes
+ vector<shared_ptr<PoolingLayer<Dtype> > > pooling_layers_;
+ /// top vector holders used in call to the underlying PoolingLayer::Forward
+ vector<vector<Blob<Dtype>*>*> pooling_top_vecs_;
+ /// pooling_outputs stores the outputs of the PoolingLayers
+ vector<Blob<Dtype>*> pooling_outputs_;
+ /// the internal Flatten layers that the Pooling layers feed into
+ vector<FlattenLayer<Dtype>*> flatten_layers_;
+ /// top vector holders used in call to the underlying FlattenLayer::Forward
+ vector<vector<Blob<Dtype>*>*> flatten_top_vecs_;
+ /// flatten_outputs stores the outputs of the FlattenLayers
+ vector<Blob<Dtype>*> flatten_outputs_;
+ /// bottom vector holder used in call to the underlying ConcatLayer::Forward
+ vector<Blob<Dtype>*> concat_bottom_vec_;
+ /// the internal Concat layers that the Flatten layers feed into
+ shared_ptr<ConcatLayer<Dtype> > concat_layer_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SPP_LAYER_HPP_
diff --git a/include/caffe/layers/tanh_layer.hpp b/include/caffe/layers/tanh_layer.hpp
new file mode 100644
index 0000000..8f95e93
--- /dev/null
+++ b/include/caffe/layers/tanh_layer.hpp
@@ -0,0 +1,73 @@
+#ifndef CAFFE_TANH_LAYER_HPP_
+#define CAFFE_TANH_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief TanH hyperbolic tangent non-linearity @f$
+ * y = \frac{\exp(2x) - 1}{\exp(2x) + 1}
+ * @f$, popular in auto-encoders.
+ *
+ * Note that the gradient vanishes as the values move away from 0.
+ * The ReLULayer is often a better choice for this reason.
+ */
+template <typename Dtype>
+class TanHLayer : public NeuronLayer<Dtype> {
+ public:
+ explicit TanHLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+
+ virtual inline const char* type() const { return "TanH"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = \frac{\exp(2x) - 1}{\exp(2x) + 1}
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ /**
+ * @brief Computes the error gradient w.r.t. the sigmoid inputs.
+ *
+ * @param top output Blob vector (length 1), providing the error gradient with
+ * respect to the outputs
+ * -# @f$ (N \times C \times H \times W) @f$
+ * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
+ * with respect to computed outputs @f$ y @f$
+ * @param propagate_down see Layer::Backward.
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$; Backward fills their diff with
+ * gradients @f$
+ * \frac{\partial E}{\partial x}
+ * = \frac{\partial E}{\partial y}
+ * \left(1 - \left[\frac{\exp(2x) - 1}{exp(2x) + 1} \right]^2 \right)
+ * = \frac{\partial E}{\partial y} (1 - y^2)
+ * @f$ if propagate_down[0]
+ */
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_TANH_LAYER_HPP_
diff --git a/include/caffe/layers/threshold_layer.hpp b/include/caffe/layers/threshold_layer.hpp
new file mode 100644
index 0000000..3bf4db6
--- /dev/null
+++ b/include/caffe/layers/threshold_layer.hpp
@@ -0,0 +1,64 @@
+#ifndef CAFFE_THRESHOLD_LAYER_HPP_
+#define CAFFE_THRESHOLD_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+#include "caffe/layers/neuron_layer.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Tests whether the input exceeds a threshold: outputs 1 for inputs
+ * above threshold; 0 otherwise.
+ */
+template <typename Dtype>
+class ThresholdLayer : public NeuronLayer<Dtype> {
+ public:
+ /**
+ * @param param provides ThresholdParameter threshold_param,
+ * with ThresholdLayer options:
+ * - threshold (\b optional, default 0).
+ * the threshold value @f$ t @f$ to which the input values are compared.
+ */
+ explicit ThresholdLayer(const LayerParameter& param)
+ : NeuronLayer<Dtype>(param) {}
+ virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Threshold"; }
+
+ protected:
+ /**
+ * @param bottom input Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the inputs @f$ x @f$
+ * @param top output Blob vector (length 1)
+ * -# @f$ (N \times C \times H \times W) @f$
+ * the computed outputs @f$
+ * y = \left\{
+ * \begin{array}{lr}
+ * 0 & \mathrm{if} \; x \le t \\
+ * 1 & \mathrm{if} \; x > t
+ * \end{array} \right.
+ * @f$
+ */
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ /// @brief Not implemented (non-differentiable function)
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
+ NOT_IMPLEMENTED;
+ }
+
+ Dtype threshold_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_THRESHOLD_LAYER_HPP_
diff --git a/include/caffe/layers/tile_layer.hpp b/include/caffe/layers/tile_layer.hpp
new file mode 100644
index 0000000..fbdbe2f
--- /dev/null
+++ b/include/caffe/layers/tile_layer.hpp
@@ -0,0 +1,43 @@
+#ifndef CAFFE_TILE_LAYER_HPP_
+#define CAFFE_TILE_LAYER_HPP_
+
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Copy a Blob along specified dimensions.
+ */
+template <typename Dtype>
+class TileLayer : public Layer<Dtype> {
+ public:
+ explicit TileLayer(const LayerParameter& param)
+ : Layer<Dtype>(param) {}
+ virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "Tile"; }
+ virtual inline int ExactNumBottomBlobs() const { return 1; }
+ virtual inline int ExactNumTopBlobs() const { return 1; }
+
+ protected:
+ virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+ virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+ virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+ unsigned int axis_, tiles_, outer_dim_, inner_dim_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_TILE_LAYER_HPP_
diff --git a/include/caffe/layers/window_data_layer.hpp b/include/caffe/layers/window_data_layer.hpp
new file mode 100644
index 0000000..35f41b8
--- /dev/null
+++ b/include/caffe/layers/window_data_layer.hpp
@@ -0,0 +1,55 @@
+#ifndef CAFFE_WINDOW_DATA_LAYER_HPP_
+#define CAFFE_WINDOW_DATA_LAYER_HPP_
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "caffe/blob.hpp"
+#include "caffe/data_transformer.hpp"
+#include "caffe/internal_thread.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/layers/base_data_layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+/**
+ * @brief Provides data to the Net from windows of images files, specified
+ * by a window data file.
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class WindowDataLayer : public BasePrefetchingDataLayer<Dtype> {
+ public:
+ explicit WindowDataLayer(const LayerParameter& param)
+ : BasePrefetchingDataLayer<Dtype>(param) {}
+ virtual ~WindowDataLayer();
+ virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top);
+
+ virtual inline const char* type() const { return "WindowData"; }
+ virtual inline int ExactNumBottomBlobs() const { return 0; }
+ virtual inline int ExactNumTopBlobs() const { return 2; }
+
+ protected:
+ virtual unsigned int PrefetchRand();
+ virtual void load_batch(Batch<Dtype>* batch);
+
+ shared_ptr<Caffe::RNG> prefetch_rng_;
+ vector<std::pair<std::string, vector<int> > > image_database_;
+ enum WindowField { IMAGE_INDEX, LABEL, OVERLAP, X1, Y1, X2, Y2, NUM };
+ vector<vector<float> > fg_windows_;
+ vector<vector<float> > bg_windows_;
+ Blob<Dtype> data_mean_;
+ vector<Dtype> mean_values_;
+ bool has_mean_file_;
+ bool has_mean_values_;
+ bool cache_images_;
+ vector<std::pair<std::string, Datum > > image_database_cache_;
+};
+
+} // namespace caffe
+
+#endif // CAFFE_WINDOW_DATA_LAYER_HPP_
diff --git a/include/caffe/loss_layers.hpp b/include/caffe/loss_layers.hpp
deleted file mode 100644
index 8d41af3..0000000
--- a/include/caffe/loss_layers.hpp
+++ /dev/null
@@ -1,773 +0,0 @@
-#ifndef CAFFE_LOSS_LAYERS_HPP_
-#define CAFFE_LOSS_LAYERS_HPP_
-
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/neuron_layers.hpp"
-#include "caffe/proto/caffe.pb.h"
-
-namespace caffe {
-
-const float kLOG_THRESHOLD = 1e-20;
-
-/**
- * @brief Computes the classification accuracy for a one-of-many
- * classification task.
- */
-template <typename Dtype>
-class AccuracyLayer : public Layer<Dtype> {
- public:
- /**
- * @param param provides AccuracyParameter accuracy_param,
- * with AccuracyLayer options:
- * - top_k (\b optional, default 1).
- * Sets the maximum rank @f$ k @f$ at which a prediction is considered
- * correct. For example, if @f$ k = 5 @f$, a prediction is counted
- * correct if the correct label is among the top 5 predicted labels.
- */
- explicit AccuracyLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Accuracy"; }
- virtual inline int ExactNumBottomBlobs() const { return 2; }
-
- // If there are two top blobs, then the second blob will contain
- // accuracies per class.
- virtual inline int MinTopBlobs() const { return 1; }
- virtual inline int MaxTopBlos() const { return 2; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ x @f$, a Blob with values in
- * @f$ [-\infty, +\infty] @f$ indicating the predicted score for each of
- * the @f$ K = CHW @f$ classes. Each @f$ x_n @f$ is mapped to a predicted
- * label @f$ \hat{l}_n @f$ given by its maximal index:
- * @f$ \hat{l}_n = \arg\max\limits_k x_{nk} @f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels @f$ l @f$, an integer-valued Blob with values
- * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
- * indicating the correct class label among the @f$ K @f$ classes
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed accuracy: @f$
- * \frac{1}{N} \sum\limits_{n=1}^N \delta\{ \hat{l}_n = l_n \}
- * @f$, where @f$
- * \delta\{\mathrm{condition}\} = \left\{
- * \begin{array}{lr}
- * 1 & \mbox{if condition} \\
- * 0 & \mbox{otherwise}
- * \end{array} \right.
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
-
- /// @brief Not implemented -- AccuracyLayer cannot be used as a loss.
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
- for (int i = 0; i < propagate_down.size(); ++i) {
- if (propagate_down[i]) { NOT_IMPLEMENTED; }
- }
- }
-
- int label_axis_, outer_num_, inner_num_;
-
- int top_k_;
-
- /// Whether to ignore instances with a certain label.
- bool has_ignore_label_;
- /// The label indicating that an instance should be ignored.
- int ignore_label_;
- /// Keeps counts of the number of samples per class.
- Blob<Dtype> nums_buffer_;
-};
-
-/**
- * @brief An interface for Layer%s that take two Blob%s as input -- usually
- * (1) predictions and (2) ground-truth labels -- and output a
- * singleton Blob representing the loss.
- *
- * LossLayers are typically only capable of backpropagating to their first input
- * -- the predictions.
- */
-template <typename Dtype>
-class LossLayer : public Layer<Dtype> {
- public:
- explicit LossLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(
- const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top);
- virtual void Reshape(
- const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top);
-
- virtual inline int ExactNumBottomBlobs() const { return 2; }
-
- /**
- * @brief For convenience and backwards compatibility, instruct the Net to
- * automatically allocate a single top Blob for LossLayers, into which
- * they output their singleton loss, (even if the user didn't specify
- * one in the prototxt, etc.).
- */
- virtual inline bool AutoTopBlobs() const { return true; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
- /**
- * We usually cannot backpropagate to the labels; ignore force_backward for
- * these inputs.
- */
- virtual inline bool AllowForceBackward(const int bottom_index) const {
- return bottom_index != 1;
- }
-};
-
-/**
- * @brief Computes the contrastive loss @f$
- * E = \frac{1}{2N} \sum\limits_{n=1}^N \left(y\right) d +
- * \left(1-y\right) \max \left(margin-d, 0\right)^2
- * @f$ where @f$
- * d = \left| \left| a_n - b_n \right| \right|_2 @f$. This can be
- * used to train siamese networks.
- *
- * @param bottom input Blob vector (length 3)
- * -# @f$ (N \times C \times 1 \times 1) @f$
- * the features @f$ a \in [-\infty, +\infty]@f$
- * -# @f$ (N \times C \times 1 \times 1) @f$
- * the features @f$ b \in [-\infty, +\infty]@f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the binary similarity @f$ s \in [0, 1]@f$
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed contrastive loss: @f$ E =
- * \frac{1}{2N} \sum\limits_{n=1}^N \left(y\right) d +
- * \left(1-y\right) \max \left(margin-d, 0\right)^2
- * @f$ where @f$
- * d = \left| \left| a_n - b_n \right| \right|_2 @f$.
- * This can be used to train siamese networks.
- */
-template <typename Dtype>
-class ContrastiveLossLayer : public LossLayer<Dtype> {
- public:
- explicit ContrastiveLossLayer(const LayerParameter& param)
- : LossLayer<Dtype>(param), diff_() {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline int ExactNumBottomBlobs() const { return 3; }
- virtual inline const char* type() const { return "ContrastiveLoss"; }
- /**
- * Unlike most loss layers, in the ContrastiveLossLayer we can backpropagate
- * to the first two inputs.
- */
- virtual inline bool AllowForceBackward(const int bottom_index) const {
- return bottom_index != 2;
- }
-
- protected:
- /// @copydoc ContrastiveLossLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the Contrastive error gradient w.r.t. the inputs.
- *
- * Computes the gradients with respect to the two input vectors (bottom[0] and
- * bottom[1]), but not the similarity label (bottom[2]).
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
- * as @f$ \lambda @f$ is the coefficient of this layer's output
- * @f$\ell_i at f$ in the overall Net loss
- * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
- * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
- * (*Assuming that this top Blob is not used as a bottom (input) by any
- * other layer of the Net.)
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times 1 \times 1) @f$
- * the features @f$a at f$; Backward fills their diff with
- * gradients if propagate_down[0]
- * -# @f$ (N \times C \times 1 \times 1) @f$
- * the features @f$b at f$; Backward fills their diff with gradients if
- * propagate_down[1]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- Blob<Dtype> diff_; // cached for backward pass
- Blob<Dtype> dist_sq_; // cached for backward pass
- Blob<Dtype> diff_sq_; // tmp storage for gpu forward pass
- Blob<Dtype> summer_vec_; // tmp storage for gpu forward pass
-};
-
-/**
- * @brief Computes the Euclidean (L2) loss @f$
- * E = \frac{1}{2N} \sum\limits_{n=1}^N \left| \left| \hat{y}_n - y_n
- * \right| \right|_2^2 @f$ for real-valued regression tasks.
- *
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ \hat{y} \in [-\infty, +\infty]@f$
- * -# @f$ (N \times C \times H \times W) @f$
- * the targets @f$ y \in [-\infty, +\infty]@f$
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed Euclidean loss: @f$ E =
- * \frac{1}{2n} \sum\limits_{n=1}^N \left| \left| \hat{y}_n - y_n
- * \right| \right|_2^2 @f$
- *
- * This can be used for least-squares regression tasks. An InnerProductLayer
- * input to a EuclideanLossLayer exactly formulates a linear least squares
- * regression problem. With non-zero weight decay the problem becomes one of
- * ridge regression -- see src/caffe/test/test_sgd_solver.cpp for a concrete
- * example wherein we check that the gradients computed for a Net with exactly
- * this structure match hand-computed gradient formulas for ridge regression.
- *
- * (Note: Caffe, and SGD in general, is certainly \b not the best way to solve
- * linear least squares problems! We use it only as an instructive example.)
- */
-template <typename Dtype>
-class EuclideanLossLayer : public LossLayer<Dtype> {
- public:
- explicit EuclideanLossLayer(const LayerParameter& param)
- : LossLayer<Dtype>(param), diff_() {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "EuclideanLoss"; }
- /**
- * Unlike most loss layers, in the EuclideanLossLayer we can backpropagate
- * to both inputs -- override to return true and always allow force_backward.
- */
- virtual inline bool AllowForceBackward(const int bottom_index) const {
- return true;
- }
-
- protected:
- /// @copydoc EuclideanLossLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the Euclidean error gradient w.r.t. the inputs.
- *
- * Unlike other children of LossLayer, EuclideanLossLayer \b can compute
- * gradients with respect to the label inputs bottom[1] (but still only will
- * if propagate_down[1] is set, due to being produced by learnable parameters
- * or if force_backward is set). In fact, this layer is "commutative" -- the
- * result is the same regardless of the order of the two bottoms.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
- * as @f$ \lambda @f$ is the coefficient of this layer's output
- * @f$\ell_i at f$ in the overall Net loss
- * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
- * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
- * (*Assuming that this top Blob is not used as a bottom (input) by any
- * other layer of the Net.)
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$\hat{y}@f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial \hat{y}} =
- * \frac{1}{n} \sum\limits_{n=1}^N (\hat{y}_n - y_n)
- * @f$ if propagate_down[0]
- * -# @f$ (N \times C \times H \times W) @f$
- * the targets @f$y at f$; Backward fills their diff with gradients
- * @f$ \frac{\partial E}{\partial y} =
- * \frac{1}{n} \sum\limits_{n=1}^N (y_n - \hat{y}_n)
- * @f$ if propagate_down[1]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- Blob<Dtype> diff_;
-};
-
-/**
- * @brief Computes the hinge loss for a one-of-many classification task.
- *
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ t @f$, a Blob with values in
- * @f$ [-\infty, +\infty] @f$ indicating the predicted score for each of
- * the @f$ K = CHW @f$ classes. In an SVM, @f$ t @f$ is the result of
- * taking the inner product @f$ X^T W @f$ of the D-dimensional features
- * @f$ X \in \mathcal{R}^{D \times N} @f$ and the learned hyperplane
- * parameters @f$ W \in \mathcal{R}^{D \times K} @f$, so a Net with just
- * an InnerProductLayer (with num_output = D) providing predictions to a
- * HingeLossLayer and no other learnable parameters or losses is
- * equivalent to an SVM.
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels @f$ l @f$, an integer-valued Blob with values
- * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
- * indicating the correct class label among the @f$ K @f$ classes
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed hinge loss: @f$ E =
- * \frac{1}{N} \sum\limits_{n=1}^N \sum\limits_{k=1}^K
- * [\max(0, 1 - \delta\{l_n = k\} t_{nk})] ^ p
- * @f$, for the @f$ L^p @f$ norm
- * (defaults to @f$ p = 1 @f$, the L1 norm; L2 norm, as in L2-SVM,
- * is also available), and @f$
- * \delta\{\mathrm{condition}\} = \left\{
- * \begin{array}{lr}
- * 1 & \mbox{if condition} \\
- * -1 & \mbox{otherwise}
- * \end{array} \right.
- * @f$
- *
- * In an SVM, @f$ t \in \mathcal{R}^{N \times K} @f$ is the result of taking
- * the inner product @f$ X^T W @f$ of the features
- * @f$ X \in \mathcal{R}^{D \times N} @f$
- * and the learned hyperplane parameters
- * @f$ W \in \mathcal{R}^{D \times K} @f$. So, a Net with just an
- * InnerProductLayer (with num_output = @f$k at f$) providing predictions to a
- * HingeLossLayer is equivalent to an SVM (assuming it has no other learned
- * outside the InnerProductLayer and no other losses outside the
- * HingeLossLayer).
- */
-template <typename Dtype>
-class HingeLossLayer : public LossLayer<Dtype> {
- public:
- explicit HingeLossLayer(const LayerParameter& param)
- : LossLayer<Dtype>(param) {}
-
- virtual inline const char* type() const { return "HingeLoss"; }
-
- protected:
- /// @copydoc HingeLossLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the hinge loss error gradient w.r.t. the predictions.
- *
- * Gradients cannot be computed with respect to the label inputs (bottom[1]),
- * so this method ignores bottom[1] and requires !propagate_down[1], crashing
- * if propagate_down[1] is set.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
- * as @f$ \lambda @f$ is the coefficient of this layer's output
- * @f$\ell_i at f$ in the overall Net loss
- * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
- * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
- * (*Assuming that this top Blob is not used as a bottom (input) by any
- * other layer of the Net.)
- * @param propagate_down see Layer::Backward.
- * propagate_down[1] must be false as we can't compute gradients with
- * respect to the labels.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$t at f$; Backward computes diff
- * @f$ \frac{\partial E}{\partial t} @f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels -- ignored as we can't compute their error gradients
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-/**
- * @brief A generalization of MultinomialLogisticLossLayer that takes an
- * "information gain" (infogain) matrix specifying the "value" of all label
- * pairs.
- *
- * Equivalent to the MultinomialLogisticLossLayer if the infogain matrix is the
- * identity.
- *
- * @param bottom input Blob vector (length 2-3)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ \hat{p} @f$, a Blob with values in
- * @f$ [0, 1] @f$ indicating the predicted probability of each of the
- * @f$ K = CHW @f$ classes. Each prediction vector @f$ \hat{p}_n @f$
- * should sum to 1 as in a probability distribution: @f$
- * \forall n \sum\limits_{k=1}^K \hat{p}_{nk} = 1 @f$.
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels @f$ l @f$, an integer-valued Blob with values
- * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
- * indicating the correct class label among the @f$ K @f$ classes
- * -# @f$ (1 \times 1 \times K \times K) @f$
- * (\b optional) the infogain matrix @f$ H @f$. This must be provided as
- * the third bottom blob input if not provided as the infogain_mat in the
- * InfogainLossParameter. If @f$ H = I @f$, this layer is equivalent to the
- * MultinomialLogisticLossLayer.
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed infogain multinomial logistic loss: @f$ E =
- * \frac{-1}{N} \sum\limits_{n=1}^N H_{l_n} \log(\hat{p}_n) =
- * \frac{-1}{N} \sum\limits_{n=1}^N \sum\limits_{k=1}^{K} H_{l_n,k}
- * \log(\hat{p}_{n,k})
- * @f$, where @f$ H_{l_n} @f$ denotes row @f$l_n at f$ of @f$H at f$.
- */
-template <typename Dtype>
-class InfogainLossLayer : public LossLayer<Dtype> {
- public:
- explicit InfogainLossLayer(const LayerParameter& param)
- : LossLayer<Dtype>(param), infogain_() {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- // InfogainLossLayer takes 2-3 bottom Blobs; if there are 3 the third should
- // be the infogain matrix. (Otherwise the infogain matrix is loaded from a
- // file specified by LayerParameter.)
- virtual inline int ExactNumBottomBlobs() const { return -1; }
- virtual inline int MinBottomBlobs() const { return 2; }
- virtual inline int MaxBottomBlobs() const { return 3; }
-
- virtual inline const char* type() const { return "InfogainLoss"; }
-
- protected:
- /// @copydoc InfogainLossLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the infogain loss error gradient w.r.t. the predictions.
- *
- * Gradients cannot be computed with respect to the label inputs (bottom[1]),
- * so this method ignores bottom[1] and requires !propagate_down[1], crashing
- * if propagate_down[1] is set. (The same applies to the infogain matrix, if
- * provided as bottom[2] rather than in the layer_param.)
- *
- * @param top output Blob vector (length 1), providing the error gradient
- * with respect to the outputs
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
- * as @f$ \lambda @f$ is the coefficient of this layer's output
- * @f$\ell_i at f$ in the overall Net loss
- * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
- * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
- * (*Assuming that this top Blob is not used as a bottom (input) by any
- * other layer of the Net.)
- * @param propagate_down see Layer::Backward.
- * propagate_down[1] must be false as we can't compute gradients with
- * respect to the labels (similarly for propagate_down[2] and the
- * infogain matrix, if provided as bottom[2])
- * @param bottom input Blob vector (length 2-3)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ \hat{p} @f$; Backward computes diff
- * @f$ \frac{\partial E}{\partial \hat{p}} @f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels -- ignored as we can't compute their error gradients
- * -# @f$ (1 \times 1 \times K \times K) @f$
- * (\b optional) the information gain matrix -- ignored as its error
- * gradient computation is not implemented.
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- Blob<Dtype> infogain_;
-};
-
-/**
- * @brief Computes the multinomial logistic loss for a one-of-many
- * classification task, directly taking a predicted probability
- * distribution as input.
- *
- * When predictions are not already a probability distribution, you should
- * instead use the SoftmaxWithLossLayer, which maps predictions to a
- * distribution using the SoftmaxLayer, before computing the multinomial
- * logistic loss. The SoftmaxWithLossLayer should be preferred over separate
- * SoftmaxLayer + MultinomialLogisticLossLayer
- * as its gradient computation is more numerically stable.
- *
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ \hat{p} @f$, a Blob with values in
- * @f$ [0, 1] @f$ indicating the predicted probability of each of the
- * @f$ K = CHW @f$ classes. Each prediction vector @f$ \hat{p}_n @f$
- * should sum to 1 as in a probability distribution: @f$
- * \forall n \sum\limits_{k=1}^K \hat{p}_{nk} = 1 @f$.
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels @f$ l @f$, an integer-valued Blob with values
- * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
- * indicating the correct class label among the @f$ K @f$ classes
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed multinomial logistic loss: @f$ E =
- * \frac{-1}{N} \sum\limits_{n=1}^N \log(\hat{p}_{n,l_n})
- * @f$
- */
-template <typename Dtype>
-class MultinomialLogisticLossLayer : public LossLayer<Dtype> {
- public:
- explicit MultinomialLogisticLossLayer(const LayerParameter& param)
- : LossLayer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "MultinomialLogisticLoss"; }
-
- protected:
- /// @copydoc MultinomialLogisticLossLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the multinomial logistic loss error gradient w.r.t. the
- * predictions.
- *
- * Gradients cannot be computed with respect to the label inputs (bottom[1]),
- * so this method ignores bottom[1] and requires !propagate_down[1], crashing
- * if propagate_down[1] is set.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
- * as @f$ \lambda @f$ is the coefficient of this layer's output
- * @f$\ell_i at f$ in the overall Net loss
- * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
- * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
- * (*Assuming that this top Blob is not used as a bottom (input) by any
- * other layer of the Net.)
- * @param propagate_down see Layer::Backward.
- * propagate_down[1] must be false as we can't compute gradients with
- * respect to the labels.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ \hat{p} @f$; Backward computes diff
- * @f$ \frac{\partial E}{\partial \hat{p}} @f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels -- ignored as we can't compute their error gradients
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-/**
- * @brief Computes the cross-entropy (logistic) loss @f$
- * E = \frac{-1}{n} \sum\limits_{n=1}^N \left[
- * p_n \log \hat{p}_n +
- * (1 - p_n) \log(1 - \hat{p}_n)
- * \right]
- * @f$, often used for predicting targets interpreted as probabilities.
- *
- * This layer is implemented rather than separate
- * SigmoidLayer + CrossEntropyLayer
- * as its gradient computation is more numerically stable.
- * At test time, this layer can be replaced simply by a SigmoidLayer.
- *
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the scores @f$ x \in [-\infty, +\infty]@f$,
- * which this layer maps to probability predictions
- * @f$ \hat{p}_n = \sigma(x_n) \in [0, 1] @f$
- * using the sigmoid function @f$ \sigma(.) @f$ (see SigmoidLayer).
- * -# @f$ (N \times C \times H \times W) @f$
- * the targets @f$ y \in [0, 1] @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed cross-entropy loss: @f$
- * E = \frac{-1}{n} \sum\limits_{n=1}^N \left[
- * p_n \log \hat{p}_n + (1 - p_n) \log(1 - \hat{p}_n)
- * \right]
- * @f$
- */
-template <typename Dtype>
-class SigmoidCrossEntropyLossLayer : public LossLayer<Dtype> {
- public:
- explicit SigmoidCrossEntropyLossLayer(const LayerParameter& param)
- : LossLayer<Dtype>(param),
- sigmoid_layer_(new SigmoidLayer<Dtype>(param)),
- sigmoid_output_(new Blob<Dtype>()) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "SigmoidCrossEntropyLoss"; }
-
- protected:
- /// @copydoc SigmoidCrossEntropyLossLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the sigmoid cross-entropy loss error gradient w.r.t. the
- * predictions.
- *
- * Gradients cannot be computed with respect to the target inputs (bottom[1]),
- * so this method ignores bottom[1] and requires !propagate_down[1], crashing
- * if propagate_down[1] is set.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
- * as @f$ \lambda @f$ is the coefficient of this layer's output
- * @f$\ell_i at f$ in the overall Net loss
- * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
- * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
- * (*Assuming that this top Blob is not used as a bottom (input) by any
- * other layer of the Net.)
- * @param propagate_down see Layer::Backward.
- * propagate_down[1] must be false as gradient computation with respect
- * to the targets is not implemented.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$x at f$; Backward computes diff
- * @f$ \frac{\partial E}{\partial x} =
- * \frac{1}{n} \sum\limits_{n=1}^N (\hat{p}_n - p_n)
- * @f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels -- ignored as we can't compute their error gradients
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- /// The internal SigmoidLayer used to map predictions to probabilities.
- shared_ptr<SigmoidLayer<Dtype> > sigmoid_layer_;
- /// sigmoid_output stores the output of the SigmoidLayer.
- shared_ptr<Blob<Dtype> > sigmoid_output_;
- /// bottom vector holder to call the underlying SigmoidLayer::Forward
- vector<Blob<Dtype>*> sigmoid_bottom_vec_;
- /// top vector holder to call the underlying SigmoidLayer::Forward
- vector<Blob<Dtype>*> sigmoid_top_vec_;
-};
-
-// Forward declare SoftmaxLayer for use in SoftmaxWithLossLayer.
-template <typename Dtype> class SoftmaxLayer;
-
-/**
- * @brief Computes the multinomial logistic loss for a one-of-many
- * classification task, passing real-valued predictions through a
- * softmax to get a probability distribution over classes.
- *
- * This layer should be preferred over separate
- * SoftmaxLayer + MultinomialLogisticLossLayer
- * as its gradient computation is more numerically stable.
- * At test time, this layer can be replaced simply by a SoftmaxLayer.
- *
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ x @f$, a Blob with values in
- * @f$ [-\infty, +\infty] @f$ indicating the predicted score for each of
- * the @f$ K = CHW @f$ classes. This layer maps these scores to a
- * probability distribution over classes using the softmax function
- * @f$ \hat{p}_{nk} = \exp(x_{nk}) /
- * \left[\sum_{k'} \exp(x_{nk'})\right] @f$ (see SoftmaxLayer).
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels @f$ l @f$, an integer-valued Blob with values
- * @f$ l_n \in [0, 1, 2, ..., K - 1] @f$
- * indicating the correct class label among the @f$ K @f$ classes
- * @param top output Blob vector (length 1)
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * the computed cross-entropy classification loss: @f$ E =
- * \frac{-1}{N} \sum\limits_{n=1}^N \log(\hat{p}_{n,l_n})
- * @f$, for softmax output class probabilites @f$ \hat{p} @f$
- */
-template <typename Dtype>
-class SoftmaxWithLossLayer : public LossLayer<Dtype> {
- public:
- /**
- * @param param provides LossParameter loss_param, with options:
- * - ignore_label (optional)
- * Specify a label value that should be ignored when computing the loss.
- * - normalize (optional, default true)
- * If true, the loss is normalized by the number of (nonignored) labels
- * present; otherwise the loss is simply summed over spatial locations.
- */
- explicit SoftmaxWithLossLayer(const LayerParameter& param)
- : LossLayer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "SoftmaxWithLoss"; }
- virtual inline int ExactNumTopBlobs() const { return -1; }
- virtual inline int MinTopBlobs() const { return 1; }
- virtual inline int MaxTopBlobs() const { return 2; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- /**
- * @brief Computes the softmax loss error gradient w.r.t. the predictions.
- *
- * Gradients cannot be computed with respect to the label inputs (bottom[1]),
- * so this method ignores bottom[1] and requires !propagate_down[1], crashing
- * if propagate_down[1] is set.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (1 \times 1 \times 1 \times 1) @f$
- * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$,
- * as @f$ \lambda @f$ is the coefficient of this layer's output
- * @f$\ell_i at f$ in the overall Net loss
- * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence
- * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$.
- * (*Assuming that this top Blob is not used as a bottom (input) by any
- * other layer of the Net.)
- * @param propagate_down see Layer::Backward.
- * propagate_down[1] must be false as we can't compute gradients with
- * respect to the labels.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the predictions @f$ x @f$; Backward computes diff
- * @f$ \frac{\partial E}{\partial x} @f$
- * -# @f$ (N \times 1 \times 1 \times 1) @f$
- * the labels -- ignored as we can't compute their error gradients
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
-
- /// The internal SoftmaxLayer used to map predictions to a distribution.
- shared_ptr<Layer<Dtype> > softmax_layer_;
- /// prob stores the output probability predictions from the SoftmaxLayer.
- Blob<Dtype> prob_;
- /// bottom vector holder used in call to the underlying SoftmaxLayer::Forward
- vector<Blob<Dtype>*> softmax_bottom_vec_;
- /// top vector holder used in call to the underlying SoftmaxLayer::Forward
- vector<Blob<Dtype>*> softmax_top_vec_;
- /// Whether to ignore instances with a certain label.
- bool has_ignore_label_;
- /// The label indicating that an instance should be ignored.
- int ignore_label_;
- /// Whether to normalize the loss by the total number of values present
- /// (otherwise just by the batch size).
- bool normalize_;
-
- int softmax_axis_, outer_num_, inner_num_;
-};
-
-} // namespace caffe
-
-#endif // CAFFE_LOSS_LAYERS_HPP_
diff --git a/include/caffe/neuron_layers.hpp b/include/caffe/neuron_layers.hpp
deleted file mode 100644
index c2e0774..0000000
--- a/include/caffe/neuron_layers.hpp
+++ /dev/null
@@ -1,809 +0,0 @@
-#ifndef CAFFE_NEURON_LAYERS_HPP_
-#define CAFFE_NEURON_LAYERS_HPP_
-
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/proto/caffe.pb.h"
-
-#define HDF5_DATA_DATASET_NAME "data"
-#define HDF5_DATA_LABEL_NAME "label"
-
-namespace caffe {
-
-/**
- * @brief An interface for layers that take one blob as input (@f$ x @f$)
- * and produce one equally-sized blob as output (@f$ y @f$), where
- * each element of the output depends only on the corresponding input
- * element.
- */
-template <typename Dtype>
-class NeuronLayer : public Layer<Dtype> {
- public:
- explicit NeuronLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-};
-
-/**
- * @brief Computes @f$ y = |x| @f$
- *
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$ y = |x| @f$
- */
-template <typename Dtype>
-class AbsValLayer : public NeuronLayer<Dtype> {
- public:
- explicit AbsValLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "AbsVal"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- /// @copydoc AbsValLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the absolute value inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x} =
- * \mathrm{sign}(x) \frac{\partial E}{\partial y}
- * @f$ if propagate_down[0]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-/**
- * @brief Computes @f$ y = x + \log(1 + \exp(-x)) @f$ if @f$ x > 0 @f$;
- * @f$ y = \log(1 + \exp(x)) @f$ otherwise.
- *
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = \left\{
- * \begin{array}{ll}
- * x + \log(1 + \exp(-x)) & \mbox{if } x > 0 \\
- * \log(1 + \exp(x)) & \mbox{otherwise}
- * \end{array} \right.
- * @f$
- */
-template <typename Dtype>
-class BNLLLayer : public NeuronLayer<Dtype> {
- public:
- explicit BNLLLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
-
- virtual inline const char* type() const { return "BNLL"; }
-
- protected:
- /// @copydoc BNLLLayer
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the BNLL inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 2)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x}
- * @f$ if propagate_down[0]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-/**
- * @brief During training only, sets a random portion of @f$x at f$ to 0, adjusting
- * the rest of the vector magnitude accordingly.
- *
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$ y = |x| @f$
- */
-template <typename Dtype>
-class DropoutLayer : public NeuronLayer<Dtype> {
- public:
- /**
- * @param param provides DropoutParameter dropout_param,
- * with DropoutLayer options:
- * - dropout_ratio (\b optional, default 0.5).
- * Sets the probability @f$ p @f$ that any given unit is dropped.
- */
- explicit DropoutLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Dropout"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs. At training time, we have @f$
- * y_{\mbox{train}} = \left\{
- * \begin{array}{ll}
- * \frac{x}{1 - p} & \mbox{if } u > p \\
- * 0 & \mbox{otherwise}
- * \end{array} \right.
- * @f$, where @f$ u \sim U(0, 1)@f$ is generated independently for each
- * input at each iteration. At test time, we simply have
- * @f$ y_{\mbox{test}} = \mathbb{E}[y_{\mbox{train}}] = x @f$.
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- /// when divided by UINT_MAX, the randomly generated values @f$u\sim U(0,1)@f$
- Blob<unsigned int> rand_vec_;
- /// the probability @f$ p @f$ of dropping any input
- Dtype threshold_;
- /// the scale for undropped inputs at train time @f$ 1 / (1 - p) @f$
- Dtype scale_;
- unsigned int uint_thres_;
-};
-
-/**
- * @brief Computes @f$ y = \gamma ^ {\alpha x + \beta} @f$,
- * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$,
- * and base @f$ \gamma @f$.
- */
-template <typename Dtype>
-class ExpLayer : public NeuronLayer<Dtype> {
- public:
- /**
- * @param param provides ExpParameter exp_param,
- * with ExpLayer options:
- * - scale (\b optional, default 1) the scale @f$ \alpha @f$
- * - shift (\b optional, default 0) the shift @f$ \beta @f$
- * - base (\b optional, default -1 for a value of @f$ e \approx 2.718 @f$)
- * the base @f$ \gamma @f$
- */
- explicit ExpLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Exp"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = \gamma ^ {\alpha x + \beta}
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the exp inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x} =
- * \frac{\partial E}{\partial y} y \alpha \log_e(gamma)
- * @f$ if propagate_down[0]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- Dtype inner_scale_, outer_scale_;
-};
-
-/**
- * @brief Computes @f$ y = log_{\gamma}(\alpha x + \beta) @f$,
- * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$,
- * and base @f$ \gamma @f$.
- */
-template <typename Dtype>
-class LogLayer : public NeuronLayer<Dtype> {
- public:
- /**
- * @param param provides LogParameter log_param,
- * with LogLayer options:
- * - scale (\b optional, default 1) the scale @f$ \alpha @f$
- * - shift (\b optional, default 0) the shift @f$ \beta @f$
- * - base (\b optional, default -1 for a value of @f$ e \approx 2.718 @f$)
- * the base @f$ \gamma @f$
- */
- explicit LogLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Log"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = log_{\gamma}(\alpha x + \beta)
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the exp inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x} =
- * \frac{\partial E}{\partial y} y \alpha \log_e(gamma)
- * @f$ if propagate_down[0]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- Dtype base_scale_;
- Dtype input_scale_, input_shift_;
- Dtype backward_num_scale_;
-};
-
-/**
- * @brief Computes @f$ y = (\alpha x + \beta) ^ \gamma @f$,
- * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$,
- * and power @f$ \gamma @f$.
- */
-template <typename Dtype>
-class PowerLayer : public NeuronLayer<Dtype> {
- public:
- /**
- * @param param provides PowerParameter power_param,
- * with PowerLayer options:
- * - scale (\b optional, default 1) the scale @f$ \alpha @f$
- * - shift (\b optional, default 0) the shift @f$ \beta @f$
- * - power (\b optional, default 1) the power @f$ \gamma @f$
- */
- explicit PowerLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Power"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = (\alpha x + \beta) ^ \gamma
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the power inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x} =
- * \frac{\partial E}{\partial y}
- * \alpha \gamma (\alpha x + \beta) ^ {\gamma - 1} =
- * \frac{\partial E}{\partial y}
- * \frac{\alpha \gamma y}{\alpha x + \beta}
- * @f$ if propagate_down[0]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- /// @brief @f$ \gamma @f$ from layer_param_.power_param()
- Dtype power_;
- /// @brief @f$ \alpha @f$ from layer_param_.power_param()
- Dtype scale_;
- /// @brief @f$ \beta @f$ from layer_param_.power_param()
- Dtype shift_;
- /// @brief Result of @f$ \alpha \gamma @f$
- Dtype diff_scale_;
-};
-
-/**
- * @brief Rectified Linear Unit non-linearity @f$ y = \max(0, x) @f$.
- * The simple max is fast to compute, and the function does not saturate.
- */
-template <typename Dtype>
-class ReLULayer : public NeuronLayer<Dtype> {
- public:
- /**
- * @param param provides ReLUParameter relu_param,
- * with ReLULayer options:
- * - negative_slope (\b optional, default 0).
- * the value @f$ \nu @f$ by which negative values are multiplied.
- */
- explicit ReLULayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
-
- virtual inline const char* type() const { return "ReLU"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = \max(0, x)
- * @f$ by default. If a non-zero negative_slope @f$ \nu @f$ is provided,
- * the computed outputs are @f$ y = \max(0, x) + \nu \min(0, x) @f$.
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the ReLU inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x} = \left\{
- * \begin{array}{lr}
- * 0 & \mathrm{if} \; x \le 0 \\
- * \frac{\partial E}{\partial y} & \mathrm{if} \; x > 0
- * \end{array} \right.
- * @f$ if propagate_down[0], by default.
- * If a non-zero negative_slope @f$ \nu @f$ is provided,
- * the computed gradients are @f$
- * \frac{\partial E}{\partial x} = \left\{
- * \begin{array}{lr}
- * \nu \frac{\partial E}{\partial y} & \mathrm{if} \; x \le 0 \\
- * \frac{\partial E}{\partial y} & \mathrm{if} \; x > 0
- * \end{array} \right.
- * @f$.
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-#ifdef USE_CUDNN
-/**
- * @brief CuDNN acceleration of ReLULayer.
- */
-template <typename Dtype>
-class CuDNNReLULayer : public ReLULayer<Dtype> {
- public:
- explicit CuDNNReLULayer(const LayerParameter& param)
- : ReLULayer<Dtype>(param), handles_setup_(false) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual ~CuDNNReLULayer();
-
- protected:
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool handles_setup_;
- cudnnHandle_t handle_;
- cudnnTensorDescriptor_t bottom_desc_;
- cudnnTensorDescriptor_t top_desc_;
-};
-#endif
-
-/**
- * @brief Sigmoid function non-linearity @f$
- * y = (1 + \exp(-x))^{-1}
- * @f$, a classic choice in neural networks.
- *
- * Note that the gradient vanishes as the values move away from 0.
- * The ReLULayer is often a better choice for this reason.
- */
-template <typename Dtype>
-class SigmoidLayer : public NeuronLayer<Dtype> {
- public:
- explicit SigmoidLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
-
- virtual inline const char* type() const { return "Sigmoid"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = (1 + \exp(-x))^{-1}
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the sigmoid inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x}
- * = \frac{\partial E}{\partial y} y (1 - y)
- * @f$ if propagate_down[0]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-#ifdef USE_CUDNN
-/**
- * @brief CuDNN acceleration of SigmoidLayer.
- */
-template <typename Dtype>
-class CuDNNSigmoidLayer : public SigmoidLayer<Dtype> {
- public:
- explicit CuDNNSigmoidLayer(const LayerParameter& param)
- : SigmoidLayer<Dtype>(param), handles_setup_(false) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual ~CuDNNSigmoidLayer();
-
- protected:
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool handles_setup_;
- cudnnHandle_t handle_;
- cudnnTensorDescriptor_t bottom_desc_;
- cudnnTensorDescriptor_t top_desc_;
-};
-#endif
-
-/**
- * @brief TanH hyperbolic tangent non-linearity @f$
- * y = \frac{\exp(2x) - 1}{\exp(2x) + 1}
- * @f$, popular in auto-encoders.
- *
- * Note that the gradient vanishes as the values move away from 0.
- * The ReLULayer is often a better choice for this reason.
- */
-template <typename Dtype>
-class TanHLayer : public NeuronLayer<Dtype> {
- public:
- explicit TanHLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
-
- virtual inline const char* type() const { return "TanH"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = \frac{\exp(2x) - 1}{\exp(2x) + 1}
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the sigmoid inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times H \times W) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$; Backward fills their diff with
- * gradients @f$
- * \frac{\partial E}{\partial x}
- * = \frac{\partial E}{\partial y}
- * \left(1 - \left[\frac{\exp(2x) - 1}{exp(2x) + 1} \right]^2 \right)
- * = \frac{\partial E}{\partial y} (1 - y^2)
- * @f$ if propagate_down[0]
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-};
-
-#ifdef USE_CUDNN
-/**
- * @brief CuDNN acceleration of TanHLayer.
- */
-template <typename Dtype>
-class CuDNNTanHLayer : public TanHLayer<Dtype> {
- public:
- explicit CuDNNTanHLayer(const LayerParameter& param)
- : TanHLayer<Dtype>(param), handles_setup_(false) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual ~CuDNNTanHLayer();
-
- protected:
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool handles_setup_;
- cudnnHandle_t handle_;
- cudnnTensorDescriptor_t bottom_desc_;
- cudnnTensorDescriptor_t top_desc_;
-};
-#endif
-
-/**
- * @brief Tests whether the input exceeds a threshold: outputs 1 for inputs
- * above threshold; 0 otherwise.
- */
-template <typename Dtype>
-class ThresholdLayer : public NeuronLayer<Dtype> {
- public:
- /**
- * @param param provides ThresholdParameter threshold_param,
- * with ThresholdLayer options:
- * - threshold (\b optional, default 0).
- * the threshold value @f$ t @f$ to which the input values are compared.
- */
- explicit ThresholdLayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Threshold"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times H \times W) @f$
- * the computed outputs @f$
- * y = \left\{
- * \begin{array}{lr}
- * 0 & \mathrm{if} \; x \le t \\
- * 1 & \mathrm{if} \; x > t
- * \end{array} \right.
- * @f$
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- /// @brief Not implemented (non-differentiable function)
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
- NOT_IMPLEMENTED;
- }
-
- Dtype threshold_;
-};
-
-/**
- * @brief Parameterized Rectified Linear Unit non-linearity @f$
- * y_i = \max(0, x_i) + a_i \min(0, x_i)
- * @f$. The differences from ReLULayer are 1) negative slopes are
- * learnable though backprop and 2) negative slopes can vary across
- * channels. The number of axes of input blob should be greater than or
- * equal to 2. The 1st axis (0-based) is seen as channels.
- */
-template <typename Dtype>
-class PReLULayer : public NeuronLayer<Dtype> {
- public:
- /**
- * @param param provides PReLUParameter prelu_param,
- * with PReLULayer options:
- * - filler (\b optional, FillerParameter,
- * default {'type': constant 'value':0.25}).
- * - channel_shared (\b optional, default false).
- * negative slopes are shared across channels.
- */
- explicit PReLULayer(const LayerParameter& param)
- : NeuronLayer<Dtype>(param) {}
-
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "PReLU"; }
-
- protected:
- /**
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times ...) @f$
- * the inputs @f$ x @f$
- * @param top output Blob vector (length 1)
- * -# @f$ (N \times C \times ...) @f$
- * the computed outputs for each channel @f$i at f$ @f$
- * y_i = \max(0, x_i) + a_i \min(0, x_i)
- * @f$.
- */
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- /**
- * @brief Computes the error gradient w.r.t. the PReLU inputs.
- *
- * @param top output Blob vector (length 1), providing the error gradient with
- * respect to the outputs
- * -# @f$ (N \times C \times ...) @f$
- * containing error gradients @f$ \frac{\partial E}{\partial y} @f$
- * with respect to computed outputs @f$ y @f$
- * @param propagate_down see Layer::Backward.
- * @param bottom input Blob vector (length 1)
- * -# @f$ (N \times C \times ...) @f$
- * the inputs @f$ x @f$; For each channel @f$i at f$, backward fills their
- * diff with gradients @f$
- * \frac{\partial E}{\partial x_i} = \left\{
- * \begin{array}{lr}
- * a_i \frac{\partial E}{\partial y_i} & \mathrm{if} \; x_i \le 0 \\
- * \frac{\partial E}{\partial y_i} & \mathrm{if} \; x_i > 0
- * \end{array} \right.
- * @f$.
- * If param_propagate_down_[0] is true, it fills the diff with gradients
- * @f$
- * \frac{\partial E}{\partial a_i} = \left\{
- * \begin{array}{lr}
- * \sum_{x_i} x_i \frac{\partial E}{\partial y_i} & \mathrm{if} \; x_i \le 0 \\
- * 0 & \mathrm{if} \; x_i > 0
- * \end{array} \right.
- * @f$.
- */
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool channel_shared_;
- Blob<Dtype> multiplier_; // dot multiplier for backward computation of params
- Blob<Dtype> backward_buff_; // temporary buffer for backward computation
- Blob<Dtype> bottom_memory_; // memory for in-place computation
-};
-
-} // namespace caffe
-
-#endif // CAFFE_NEURON_LAYERS_HPP_
diff --git a/include/caffe/sgd_solvers.hpp b/include/caffe/sgd_solvers.hpp
new file mode 100644
index 0000000..1fc52d8
--- /dev/null
+++ b/include/caffe/sgd_solvers.hpp
@@ -0,0 +1,148 @@
+#ifndef CAFFE_SGD_SOLVERS_HPP_
+#define CAFFE_SGD_SOLVERS_HPP_
+
+#include <string>
+#include <vector>
+
+#include "caffe/solver.hpp"
+
+namespace caffe {
+
+/**
+ * @brief Optimizes the parameters of a Net using
+ * stochastic gradient descent (SGD) with momentum.
+ */
+template <typename Dtype>
+class SGDSolver : public Solver<Dtype> {
+ public:
+ explicit SGDSolver(const SolverParameter& param)
+ : Solver<Dtype>(param) { PreSolve(); }
+ explicit SGDSolver(const string& param_file)
+ : Solver<Dtype>(param_file) { PreSolve(); }
+ virtual inline const char* type() const { return "SGD"; }
+
+ const vector<shared_ptr<Blob<Dtype> > >& history() { return history_; }
+
+ protected:
+ void PreSolve();
+ Dtype GetLearningRate();
+ virtual void ApplyUpdate();
+ virtual void Normalize(int param_id);
+ virtual void Regularize(int param_id);
+ virtual void ComputeUpdateValue(int param_id, Dtype rate);
+ virtual void ClipGradients();
+ virtual void SnapshotSolverState(const string& model_filename);
+ virtual void SnapshotSolverStateToBinaryProto(const string& model_filename);
+ virtual void SnapshotSolverStateToHDF5(const string& model_filename);
+ virtual void RestoreSolverStateFromHDF5(const string& state_file);
+ virtual void RestoreSolverStateFromBinaryProto(const string& state_file);
+ // history maintains the historical momentum data.
+ // update maintains update related data and is not needed in snapshots.
+ // temp maintains other information that might be needed in computation
+ // of gradients/updates and is not needed in snapshots
+ vector<shared_ptr<Blob<Dtype> > > history_, update_, temp_;
+
+ DISABLE_COPY_AND_ASSIGN(SGDSolver);
+};
+
+template <typename Dtype>
+class NesterovSolver : public SGDSolver<Dtype> {
+ public:
+ explicit NesterovSolver(const SolverParameter& param)
+ : SGDSolver<Dtype>(param) {}
+ explicit NesterovSolver(const string& param_file)
+ : SGDSolver<Dtype>(param_file) {}
+ virtual inline const char* type() const { return "Nesterov"; }
+
+ protected:
+ virtual void ComputeUpdateValue(int param_id, Dtype rate);
+
+ DISABLE_COPY_AND_ASSIGN(NesterovSolver);
+};
+
+template <typename Dtype>
+class AdaGradSolver : public SGDSolver<Dtype> {
+ public:
+ explicit AdaGradSolver(const SolverParameter& param)
+ : SGDSolver<Dtype>(param) { constructor_sanity_check(); }
+ explicit AdaGradSolver(const string& param_file)
+ : SGDSolver<Dtype>(param_file) { constructor_sanity_check(); }
+ virtual inline const char* type() const { return "AdaGrad"; }
+
+ protected:
+ virtual void ComputeUpdateValue(int param_id, Dtype rate);
+ void constructor_sanity_check() {
+ CHECK_EQ(0, this->param_.momentum())
+ << "Momentum cannot be used with AdaGrad.";
+ }
+
+ DISABLE_COPY_AND_ASSIGN(AdaGradSolver);
+};
+
+
+template <typename Dtype>
+class RMSPropSolver : public SGDSolver<Dtype> {
+ public:
+ explicit RMSPropSolver(const SolverParameter& param)
+ : SGDSolver<Dtype>(param) { constructor_sanity_check(); }
+ explicit RMSPropSolver(const string& param_file)
+ : SGDSolver<Dtype>(param_file) { constructor_sanity_check(); }
+ virtual inline const char* type() const { return "RMSProp"; }
+
+ protected:
+ virtual void ComputeUpdateValue(int param_id, Dtype rate);
+ void constructor_sanity_check() {
+ CHECK_EQ(0, this->param_.momentum())
+ << "Momentum cannot be used with RMSProp.";
+ CHECK_GE(this->param_.rms_decay(), 0)
+ << "rms_decay should lie between 0 and 1.";
+ CHECK_LT(this->param_.rms_decay(), 1)
+ << "rms_decay should lie between 0 and 1.";
+ }
+
+ DISABLE_COPY_AND_ASSIGN(RMSPropSolver);
+};
+
+template <typename Dtype>
+class AdaDeltaSolver : public SGDSolver<Dtype> {
+ public:
+ explicit AdaDeltaSolver(const SolverParameter& param)
+ : SGDSolver<Dtype>(param) { AdaDeltaPreSolve(); }
+ explicit AdaDeltaSolver(const string& param_file)
+ : SGDSolver<Dtype>(param_file) { AdaDeltaPreSolve(); }
+ virtual inline const char* type() const { return "AdaDelta"; }
+
+ protected:
+ void AdaDeltaPreSolve();
+ virtual void ComputeUpdateValue(int param_id, Dtype rate);
+
+ DISABLE_COPY_AND_ASSIGN(AdaDeltaSolver);
+};
+
+/**
+ * @brief AdamSolver, an algorithm for first-order gradient-based optimization
+ * of stochastic objective functions, based on adaptive estimates of
+ * lower-order moments. Described in [1].
+ *
+ * [1] D. P. Kingma and J. L. Ba, "ADAM: A Method for Stochastic Optimization."
+ * arXiv preprint arXiv:1412.6980v8 (2014).
+ */
+template <typename Dtype>
+class AdamSolver : public SGDSolver<Dtype> {
+ public:
+ explicit AdamSolver(const SolverParameter& param)
+ : SGDSolver<Dtype>(param) { AdamPreSolve();}
+ explicit AdamSolver(const string& param_file)
+ : SGDSolver<Dtype>(param_file) { AdamPreSolve(); }
+ virtual inline const char* type() const { return "Adam"; }
+
+ protected:
+ void AdamPreSolve();
+ virtual void ComputeUpdateValue(int param_id, Dtype rate);
+
+ DISABLE_COPY_AND_ASSIGN(AdamSolver);
+};
+
+} // namespace caffe
+
+#endif // CAFFE_SGD_SOLVERS_HPP_
diff --git a/include/caffe/solver.hpp b/include/caffe/solver.hpp
index 2ecf539..26b8e8e 100644
--- a/include/caffe/solver.hpp
+++ b/include/caffe/solver.hpp
@@ -1,10 +1,11 @@
-#ifndef CAFFE_OPTIMIZATION_SOLVER_HPP_
-#define CAFFE_OPTIMIZATION_SOLVER_HPP_
+#ifndef CAFFE_SOLVER_HPP_
+#define CAFFE_SOLVER_HPP_
#include <boost/function.hpp>
#include <string>
#include <vector>
#include "caffe/net.hpp"
+#include "caffe/solver_factory.hpp"
namespace caffe {
@@ -60,6 +61,11 @@ class Solver {
// RestoreSolverStateFrom___ protected methods. You should implement these
// methods to restore the state from the appropriate snapshot type.
void Restore(const char* resume_file);
+ // The Solver::Snapshot function implements the basic snapshotting utility
+ // that stores the learned net. You should implement the SnapshotSolverState()
+ // function that produces a SolverState protocol buffer that needs to be
+ // written to disk together with the learned net.
+ void Snapshot();
virtual ~Solver() {}
inline const SolverParameter& param() const { return param_; }
inline shared_ptr<Net<Dtype> > net() { return net_; }
@@ -83,15 +89,14 @@ class Solver {
}
void CheckSnapshotWritePermissions();
+ /**
+ * @brief Returns the solver type.
+ */
+ virtual inline const char* type() const { return ""; }
protected:
// Make and apply the update value for the current iteration.
virtual void ApplyUpdate() = 0;
- // The Solver::Snapshot function implements the basic snapshotting utility
- // that stores the learned net. You should implement the SnapshotSolverState()
- // function that produces a SolverState protocol buffer that needs to be
- // written to disk together with the learned net.
- void Snapshot();
string SnapshotFilename(const string extension);
string SnapshotToBinaryProto();
string SnapshotToHDF5();
@@ -148,158 +153,6 @@ class WorkerSolver : public Solver<Dtype> {
}
};
-/**
- * @brief Optimizes the parameters of a Net using
- * stochastic gradient descent (SGD) with momentum.
- */
-template <typename Dtype>
-class SGDSolver : public Solver<Dtype> {
- public:
- explicit SGDSolver(const SolverParameter& param)
- : Solver<Dtype>(param) { PreSolve(); }
- explicit SGDSolver(const string& param_file)
- : Solver<Dtype>(param_file) { PreSolve(); }
-
- const vector<shared_ptr<Blob<Dtype> > >& history() { return history_; }
-
- protected:
- void PreSolve();
- Dtype GetLearningRate();
- virtual void ApplyUpdate();
- virtual void Normalize(int param_id);
- virtual void Regularize(int param_id);
- virtual void ComputeUpdateValue(int param_id, Dtype rate);
- virtual void ClipGradients();
- virtual void SnapshotSolverState(const string& model_filename);
- virtual void SnapshotSolverStateToBinaryProto(const string& model_filename);
- virtual void SnapshotSolverStateToHDF5(const string& model_filename);
- virtual void RestoreSolverStateFromHDF5(const string& state_file);
- virtual void RestoreSolverStateFromBinaryProto(const string& state_file);
- // history maintains the historical momentum data.
- // update maintains update related data and is not needed in snapshots.
- // temp maintains other information that might be needed in computation
- // of gradients/updates and is not needed in snapshots
- vector<shared_ptr<Blob<Dtype> > > history_, update_, temp_;
-
- DISABLE_COPY_AND_ASSIGN(SGDSolver);
-};
-
-template <typename Dtype>
-class NesterovSolver : public SGDSolver<Dtype> {
- public:
- explicit NesterovSolver(const SolverParameter& param)
- : SGDSolver<Dtype>(param) {}
- explicit NesterovSolver(const string& param_file)
- : SGDSolver<Dtype>(param_file) {}
-
- protected:
- virtual void ComputeUpdateValue(int param_id, Dtype rate);
-
- DISABLE_COPY_AND_ASSIGN(NesterovSolver);
-};
-
-template <typename Dtype>
-class AdaGradSolver : public SGDSolver<Dtype> {
- public:
- explicit AdaGradSolver(const SolverParameter& param)
- : SGDSolver<Dtype>(param) { constructor_sanity_check(); }
- explicit AdaGradSolver(const string& param_file)
- : SGDSolver<Dtype>(param_file) { constructor_sanity_check(); }
-
- protected:
- virtual void ComputeUpdateValue(int param_id, Dtype rate);
- void constructor_sanity_check() {
- CHECK_EQ(0, this->param_.momentum())
- << "Momentum cannot be used with AdaGrad.";
- }
-
- DISABLE_COPY_AND_ASSIGN(AdaGradSolver);
-};
-
-
-template <typename Dtype>
-class RMSPropSolver : public SGDSolver<Dtype> {
- public:
- explicit RMSPropSolver(const SolverParameter& param)
- : SGDSolver<Dtype>(param) { constructor_sanity_check(); }
- explicit RMSPropSolver(const string& param_file)
- : SGDSolver<Dtype>(param_file) { constructor_sanity_check(); }
-
- protected:
- virtual void ComputeUpdateValue(int param_id, Dtype rate);
- void constructor_sanity_check() {
- CHECK_EQ(0, this->param_.momentum())
- << "Momentum cannot be used with RMSProp.";
- CHECK_GE(this->param_.rms_decay(), 0)
- << "rms_decay should lie between 0 and 1.";
- CHECK_LT(this->param_.rms_decay(), 1)
- << "rms_decay should lie between 0 and 1.";
- }
-
- DISABLE_COPY_AND_ASSIGN(RMSPropSolver);
-};
-
-template <typename Dtype>
-class AdaDeltaSolver : public SGDSolver<Dtype> {
- public:
- explicit AdaDeltaSolver(const SolverParameter& param)
- : SGDSolver<Dtype>(param) { AdaDeltaPreSolve(); }
- explicit AdaDeltaSolver(const string& param_file)
- : SGDSolver<Dtype>(param_file) { AdaDeltaPreSolve(); }
-
- protected:
- void AdaDeltaPreSolve();
- virtual void ComputeUpdateValue(int param_id, Dtype rate);
-
- DISABLE_COPY_AND_ASSIGN(AdaDeltaSolver);
-};
-
-/**
- * @brief AdamSolver, an algorithm for first-order gradient-based optimization
- * of stochastic objective functions, based on adaptive estimates of
- * lower-order moments. Described in [1].
- *
- * [1] D. P. Kingma and J. L. Ba, "ADAM: A Method for Stochastic Optimization."
- * arXiv preprint arXiv:1412.6980v8 (2014).
- */
-template <typename Dtype>
-class AdamSolver : public SGDSolver<Dtype> {
- public:
- explicit AdamSolver(const SolverParameter& param)
- : SGDSolver<Dtype>(param) { AdamPreSolve();}
- explicit AdamSolver(const string& param_file)
- : SGDSolver<Dtype>(param_file) { AdamPreSolve(); }
-
- protected:
- void AdamPreSolve();
- virtual void ComputeUpdateValue(int param_id, Dtype rate);
-
- DISABLE_COPY_AND_ASSIGN(AdamSolver);
-};
-
-template <typename Dtype>
-Solver<Dtype>* GetSolver(const SolverParameter& param) {
- SolverParameter_SolverType type = param.solver_type();
-
- switch (type) {
- case SolverParameter_SolverType_SGD:
- return new SGDSolver<Dtype>(param);
- case SolverParameter_SolverType_NESTEROV:
- return new NesterovSolver<Dtype>(param);
- case SolverParameter_SolverType_ADAGRAD:
- return new AdaGradSolver<Dtype>(param);
- case SolverParameter_SolverType_RMSPROP:
- return new RMSPropSolver<Dtype>(param);
- case SolverParameter_SolverType_ADADELTA:
- return new AdaDeltaSolver<Dtype>(param);
- case SolverParameter_SolverType_ADAM:
- return new AdamSolver<Dtype>(param);
- default:
- LOG(FATAL) << "Unknown SolverType: " << type;
- }
- return (Solver<Dtype>*) NULL;
-}
-
} // namespace caffe
-#endif // CAFFE_OPTIMIZATION_SOLVER_HPP_
+#endif // CAFFE_SOLVER_HPP_
diff --git a/include/caffe/solver_factory.hpp b/include/caffe/solver_factory.hpp
new file mode 100644
index 0000000..cfff721
--- /dev/null
+++ b/include/caffe/solver_factory.hpp
@@ -0,0 +1,137 @@
+/**
+ * @brief A solver factory that allows one to register solvers, similar to
+ * layer factory. During runtime, registered solvers could be called by passing
+ * a SolverParameter protobuffer to the CreateSolver function:
+ *
+ * SolverRegistry<Dtype>::CreateSolver(param);
+ *
+ * There are two ways to register a solver. Assuming that we have a solver like:
+ *
+ * template <typename Dtype>
+ * class MyAwesomeSolver : public Solver<Dtype> {
+ * // your implementations
+ * };
+ *
+ * and its type is its C++ class name, but without the "Solver" at the end
+ * ("MyAwesomeSolver" -> "MyAwesome").
+ *
+ * If the solver is going to be created simply by its constructor, in your c++
+ * file, add the following line:
+ *
+ * REGISTER_SOLVER_CLASS(MyAwesome);
+ *
+ * Or, if the solver is going to be created by another creator function, in the
+ * format of:
+ *
+ * template <typename Dtype>
+ * Solver<Dtype*> GetMyAwesomeSolver(const SolverParameter& param) {
+ * // your implementation
+ * }
+ *
+ * then you can register the creator function instead, like
+ *
+ * REGISTER_SOLVER_CREATOR(MyAwesome, GetMyAwesomeSolver)
+ *
+ * Note that each solver type should only be registered once.
+ */
+
+#ifndef CAFFE_SOLVER_FACTORY_H_
+#define CAFFE_SOLVER_FACTORY_H_
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "caffe/common.hpp"
+#include "caffe/proto/caffe.pb.h"
+
+namespace caffe {
+
+template <typename Dtype>
+class Solver;
+
+template <typename Dtype>
+class SolverRegistry {
+ public:
+ typedef Solver<Dtype>* (*Creator)(const SolverParameter&);
+ typedef std::map<string, Creator> CreatorRegistry;
+
+ static CreatorRegistry& Registry() {
+ static CreatorRegistry* g_registry_ = new CreatorRegistry();
+ return *g_registry_;
+ }
+
+ // Adds a creator.
+ static void AddCreator(const string& type, Creator creator) {
+ CreatorRegistry& registry = Registry();
+ CHECK_EQ(registry.count(type), 0)
+ << "Solver type " << type << " already registered.";
+ registry[type] = creator;
+ }
+
+ // Get a solver using a SolverParameter.
+ static Solver<Dtype>* CreateSolver(const SolverParameter& param) {
+ const string& type = param.type();
+ CreatorRegistry& registry = Registry();
+ CHECK_EQ(registry.count(type), 1) << "Unknown solver type: " << type
+ << " (known types: " << SolverTypeListString() << ")";
+ return registry[type](param);
+ }
+
+ static vector<string> SolverTypeList() {
+ CreatorRegistry& registry = Registry();
+ vector<string> solver_types;
+ for (typename CreatorRegistry::iterator iter = registry.begin();
+ iter != registry.end(); ++iter) {
+ solver_types.push_back(iter->first);
+ }
+ return solver_types;
+ }
+
+ private:
+ // Solver registry should never be instantiated - everything is done with its
+ // static variables.
+ SolverRegistry() {}
+
+ static string SolverTypeListString() {
+ vector<string> solver_types = SolverTypeList();
+ string solver_types_str;
+ for (vector<string>::iterator iter = solver_types.begin();
+ iter != solver_types.end(); ++iter) {
+ if (iter != solver_types.begin()) {
+ solver_types_str += ", ";
+ }
+ solver_types_str += *iter;
+ }
+ return solver_types_str;
+ }
+};
+
+
+template <typename Dtype>
+class SolverRegisterer {
+ public:
+ SolverRegisterer(const string& type,
+ Solver<Dtype>* (*creator)(const SolverParameter&)) {
+ // LOG(INFO) << "Registering solver type: " << type;
+ SolverRegistry<Dtype>::AddCreator(type, creator);
+ }
+};
+
+
+#define REGISTER_SOLVER_CREATOR(type, creator) \
+ static SolverRegisterer<float> g_creator_f_##type(#type, creator<float>); \
+ static SolverRegisterer<double> g_creator_d_##type(#type, creator<double>) \
+
+#define REGISTER_SOLVER_CLASS(type) \
+ template <typename Dtype> \
+ Solver<Dtype>* Creator_##type##Solver( \
+ const SolverParameter& param) \
+ { \
+ return new type##Solver<Dtype>(param); \
+ } \
+ REGISTER_SOLVER_CREATOR(type, Creator_##type##Solver)
+
+} // namespace caffe
+
+#endif // CAFFE_SOLVER_FACTORY_H_
diff --git a/include/caffe/syncedmem.hpp b/include/caffe/syncedmem.hpp
index 62aadef..38ee466 100644
--- a/include/caffe/syncedmem.hpp
+++ b/include/caffe/syncedmem.hpp
@@ -4,7 +4,6 @@
#include <cstdlib>
#include "caffe/common.hpp"
-#include "caffe/util/math_functions.hpp"
namespace caffe {
@@ -13,20 +12,22 @@ namespace caffe {
// The improvement in performance seems negligible in the single GPU case,
// but might be more significant for parallel training. Most importantly,
// it improved stability for large models on many GPUs.
-inline void CaffeMallocHost(void** ptr, size_t size) {
+inline void CaffeMallocHost(void** ptr, size_t size, bool* use_cuda) {
#ifndef CPU_ONLY
if (Caffe::mode() == Caffe::GPU) {
CUDA_CHECK(cudaMallocHost(ptr, size));
+ *use_cuda = true;
return;
}
#endif
*ptr = malloc(size);
+ *use_cuda = false;
CHECK(*ptr) << "host allocation of size " << size << " failed";
}
-inline void CaffeFreeHost(void* ptr) {
+inline void CaffeFreeHost(void* ptr, bool use_cuda) {
#ifndef CPU_ONLY
- if (Caffe::mode() == Caffe::GPU) {
+ if (use_cuda) {
CUDA_CHECK(cudaFreeHost(ptr));
return;
}
@@ -45,10 +46,12 @@ class SyncedMemory {
public:
SyncedMemory()
: cpu_ptr_(NULL), gpu_ptr_(NULL), size_(0), head_(UNINITIALIZED),
- own_cpu_data_(false), own_gpu_data_(false), gpu_device_(-1) {}
+ own_cpu_data_(false), cpu_malloc_use_cuda_(false), own_gpu_data_(false),
+ gpu_device_(-1) {}
explicit SyncedMemory(size_t size)
: cpu_ptr_(NULL), gpu_ptr_(NULL), size_(size), head_(UNINITIALIZED),
- own_cpu_data_(false), own_gpu_data_(false), gpu_device_(-1) {}
+ own_cpu_data_(false), cpu_malloc_use_cuda_(false), own_gpu_data_(false),
+ gpu_device_(-1) {}
~SyncedMemory();
const void* cpu_data();
void set_cpu_data(void* data);
@@ -72,6 +75,7 @@ class SyncedMemory {
size_t size_;
SyncedHead head_;
bool own_cpu_data_;
+ bool cpu_malloc_use_cuda_;
bool own_gpu_data_;
int gpu_device_;
diff --git a/include/caffe/test/test_gradient_check_util.hpp b/include/caffe/test/test_gradient_check_util.hpp
index 25f35d1..b25a848 100644
--- a/include/caffe/test/test_gradient_check_util.hpp
+++ b/include/caffe/test/test_gradient_check_util.hpp
@@ -169,8 +169,9 @@ void GradientChecker<Dtype>::CheckGradientSingle(Layer<Dtype>* layer,
|| fabs(feature) > kink_ + kink_range_) {
// We check relative accuracy, but for too small values, we threshold
// the scale factor by 1.
- Dtype scale = std::max(
- std::max(fabs(computed_gradient), fabs(estimated_gradient)), 1.);
+ Dtype scale = std::max<Dtype>(
+ std::max(fabs(computed_gradient), fabs(estimated_gradient)),
+ Dtype(1.));
EXPECT_NEAR(computed_gradient, estimated_gradient, threshold_ * scale)
<< "debug: (top_id, top_data_id, blob_id, feat_id)="
<< top_id << "," << top_data_id << "," << blob_id << "," << feat_id
diff --git a/include/caffe/util/blocking_queue.hpp b/include/caffe/util/blocking_queue.hpp
index 955e12c..d3de2e5 100644
--- a/include/caffe/util/blocking_queue.hpp
+++ b/include/caffe/util/blocking_queue.hpp
@@ -4,8 +4,6 @@
#include <queue>
#include <string>
-#include "caffe/common.hpp"
-
namespace caffe {
template<typename T>
diff --git a/include/caffe/util/cudnn.hpp b/include/caffe/util/cudnn.hpp
index b531dd5..8a7e17c 100644
--- a/include/caffe/util/cudnn.hpp
+++ b/include/caffe/util/cudnn.hpp
@@ -7,6 +7,9 @@
#include "caffe/common.hpp"
#include "caffe/proto/caffe.pb.h"
+#define CUDNN_VERSION_MIN(major, minor, patch) \
+ (CUDNN_VERSION >= (major * 1000 + minor * 100 + patch))
+
#define CUDNN_CHECK(condition) \
do { \
cudnnStatus_t status = condition; \
diff --git a/include/caffe/util/device_alternate.hpp b/include/caffe/util/device_alternate.hpp
index 6ea595d..e3fe4fe 100644
--- a/include/caffe/util/device_alternate.hpp
+++ b/include/caffe/util/device_alternate.hpp
@@ -81,14 +81,8 @@ namespace caffe {
const char* cublasGetErrorString(cublasStatus_t error);
const char* curandGetErrorString(curandStatus_t error);
-// CUDA: thread number configuration.
-// Use 1024 threads per block, which requires cuda sm_2x or above,
-// or fall back to attempt compatibility (best of luck to you).
-#if __CUDA_ARCH__ >= 200
- const int CAFFE_CUDA_NUM_THREADS = 1024;
-#else
- const int CAFFE_CUDA_NUM_THREADS = 512;
-#endif
+// CUDA: use 512 threads per block
+const int CAFFE_CUDA_NUM_THREADS = 512;
// CUDA: number of blocks for threads.
inline int CAFFE_GET_BLOCKS(const int N) {
diff --git a/include/caffe/util/format.hpp b/include/caffe/util/format.hpp
new file mode 100644
index 0000000..925ad2e
--- /dev/null
+++ b/include/caffe/util/format.hpp
@@ -0,0 +1,18 @@
+#ifndef CAFFE_UTIL_FORMAT_H_
+#define CAFFE_UTIL_FORMAT_H_
+
+#include <iomanip> // NOLINT(readability/streams)
+#include <sstream> // NOLINT(readability/streams)
+#include <string>
+
+namespace caffe {
+
+inline std::string format_int(int n, int numberOfLeadingZeros = 0 ) {
+ std::ostringstream s;
+ s << std::setw(numberOfLeadingZeros) << std::setfill('0') << n;
+ return s.str();
+}
+
+}
+
+#endif // CAFFE_UTIL_FORMAT_H_
diff --git a/include/caffe/util/im2col.hpp b/include/caffe/util/im2col.hpp
index 531fd29..d3eb6cc 100644
--- a/include/caffe/util/im2col.hpp
+++ b/include/caffe/util/im2col.hpp
@@ -23,7 +23,7 @@ void col2im_nd_cpu(const Dtype* data_col, const int num_spatial_axes,
template <typename Dtype>
void col2im_cpu(const Dtype* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w, const int stride_h,
const int stride_w, Dtype* data_im);
@@ -47,7 +47,7 @@ void col2im_nd_gpu(const Dtype* data_col, const int num_spatial_axes,
template <typename Dtype>
void col2im_gpu(const Dtype* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w, const int stride_h,
const int stride_w, Dtype* data_im);
diff --git a/include/caffe/util/io.hpp b/include/caffe/util/io.hpp
index 6070b4c..1a59988 100644
--- a/include/caffe/util/io.hpp
+++ b/include/caffe/util/io.hpp
@@ -1,43 +1,52 @@
#ifndef CAFFE_UTIL_IO_H_
#define CAFFE_UTIL_IO_H_
-#include <unistd.h>
+#include <boost/filesystem.hpp>
+#include <iomanip>
+#include <iostream> // NOLINT(readability/streams)
#include <string>
#include "google/protobuf/message.h"
-#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/proto/caffe.pb.h"
+#include "caffe/util/format.hpp"
+
+#ifndef CAFFE_TMP_DIR_RETRIES
+#define CAFFE_TMP_DIR_RETRIES 100
+#endif
namespace caffe {
using ::google::protobuf::Message;
-
-inline void MakeTempFilename(string* temp_filename) {
- temp_filename->clear();
- *temp_filename = "/tmp/caffe_test.XXXXXX";
- char* temp_filename_cstr = new char[temp_filename->size() + 1];
- // NOLINT_NEXT_LINE(runtime/printf)
- strcpy(temp_filename_cstr, temp_filename->c_str());
- int fd = mkstemp(temp_filename_cstr);
- CHECK_GE(fd, 0) << "Failed to open a temporary file at: " << *temp_filename;
- close(fd);
- *temp_filename = temp_filename_cstr;
- delete[] temp_filename_cstr;
-}
+using ::boost::filesystem::path;
inline void MakeTempDir(string* temp_dirname) {
temp_dirname->clear();
- *temp_dirname = "/tmp/caffe_test.XXXXXX";
- char* temp_dirname_cstr = new char[temp_dirname->size() + 1];
- // NOLINT_NEXT_LINE(runtime/printf)
- strcpy(temp_dirname_cstr, temp_dirname->c_str());
- char* mkdtemp_result = mkdtemp(temp_dirname_cstr);
- CHECK(mkdtemp_result != NULL)
- << "Failed to create a temporary directory at: " << *temp_dirname;
- *temp_dirname = temp_dirname_cstr;
- delete[] temp_dirname_cstr;
+ const path& model =
+ boost::filesystem::temp_directory_path()/"caffe_test.%%%%-%%%%";
+ for ( int i = 0; i < CAFFE_TMP_DIR_RETRIES; i++ ) {
+ const path& dir = boost::filesystem::unique_path(model).string();
+ bool done = boost::filesystem::create_directory(dir);
+ if ( done ) {
+ *temp_dirname = dir.string();
+ return;
+ }
+ }
+ LOG(FATAL) << "Failed to create a temporary directory.";
+}
+
+inline void MakeTempFilename(string* temp_filename) {
+ static path temp_files_subpath;
+ static uint64_t next_temp_file = 0;
+ temp_filename->clear();
+ if ( temp_files_subpath.empty() ) {
+ string path_string="";
+ MakeTempDir(&path_string);
+ temp_files_subpath = path_string;
+ }
+ *temp_filename =
+ (temp_files_subpath/caffe::format_int(next_temp_file++, 9)).string();
}
bool ReadProtoFromTextFile(const char* filename, Message* proto);
diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp
index 2cacd8e..6f6d3fe 100644
--- a/include/caffe/util/math_functions.hpp
+++ b/include/caffe/util/math_functions.hpp
@@ -101,9 +101,6 @@ template <typename Dtype>
Dtype caffe_cpu_strided_dot(const int n, const Dtype* x, const int incx,
const Dtype* y, const int incy);
-template <typename Dtype>
-int caffe_cpu_hamming_distance(const int n, const Dtype* x, const Dtype* y);
-
// Returns the sum of the absolute values of the elements of vector x
template <typename Dtype>
Dtype caffe_cpu_asum(const int n, const Dtype* x);
@@ -235,10 +232,6 @@ template <typename Dtype>
void caffe_gpu_dot(const int n, const Dtype* x, const Dtype* y, Dtype* out);
template <typename Dtype>
-uint32_t caffe_gpu_hamming_distance(const int n, const Dtype* x,
- const Dtype* y);
-
-template <typename Dtype>
void caffe_gpu_asum(const int n, const Dtype* x, Dtype* y);
template<typename Dtype>
diff --git a/include/caffe/util/upgrade_proto.hpp b/include/caffe/util/upgrade_proto.hpp
index c1f21a0..c94bb3c 100644
--- a/include/caffe/util/upgrade_proto.hpp
+++ b/include/caffe/util/upgrade_proto.hpp
@@ -10,6 +10,15 @@ namespace caffe {
// Return true iff the net is not the current version.
bool NetNeedsUpgrade(const NetParameter& net_param);
+// Check for deprecations and upgrade the NetParameter as needed.
+bool UpgradeNetAsNeeded(const string& param_file, NetParameter* param);
+
+// Read parameters from a file into a NetParameter proto message.
+void ReadNetParamsFromTextFileOrDie(const string& param_file,
+ NetParameter* param);
+void ReadNetParamsFromBinaryFileOrDie(const string& param_file,
+ NetParameter* param);
+
// Return true iff any layer contains parameters specified using
// deprecated V0LayerParameter.
bool NetNeedsV0ToV1Upgrade(const NetParameter& net_param);
@@ -50,14 +59,17 @@ bool UpgradeV1LayerParameter(const V1LayerParameter& v1_layer_param,
const char* UpgradeV1LayerType(const V1LayerParameter_LayerType type);
-// Check for deprecations and upgrade the NetParameter as needed.
-bool UpgradeNetAsNeeded(const string& param_file, NetParameter* param);
+// Return true iff the solver contains any old solver_type specified as enums
+bool SolverNeedsTypeUpgrade(const SolverParameter& solver_param);
-// Read parameters from a file into a NetParameter proto message.
-void ReadNetParamsFromTextFileOrDie(const string& param_file,
- NetParameter* param);
-void ReadNetParamsFromBinaryFileOrDie(const string& param_file,
- NetParameter* param);
+bool UpgradeSolverType(SolverParameter* solver_param);
+
+// Check for deprecations and upgrade the SolverParameter as needed.
+bool UpgradeSolverAsNeeded(const string& param_file, SolverParameter* param);
+
+// Read parameters from a file into a SolverParameter proto message.
+void ReadSolverParamsFromTextFileOrDie(const string& param_file,
+ SolverParameter* param);
} // namespace caffe
diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp
deleted file mode 100644
index 06bc045..0000000
--- a/include/caffe/vision_layers.hpp
+++ /dev/null
@@ -1,589 +0,0 @@
-#ifndef CAFFE_VISION_LAYERS_HPP_
-#define CAFFE_VISION_LAYERS_HPP_
-
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/common_layers.hpp"
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/loss_layers.hpp"
-#include "caffe/neuron_layers.hpp"
-#include "caffe/proto/caffe.pb.h"
-
-namespace caffe {
-
-/**
- * @brief Abstract base class that factors out the BLAS code common to
- * ConvolutionLayer and DeconvolutionLayer.
- */
-template <typename Dtype>
-class BaseConvolutionLayer : public Layer<Dtype> {
- public:
- explicit BaseConvolutionLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline int MinBottomBlobs() const { return 1; }
- virtual inline int MinTopBlobs() const { return 1; }
- virtual inline bool EqualNumBottomTopBlobs() const { return true; }
-
- protected:
- // Helper functions that abstract away the column buffer and gemm arguments.
- // The last argument in forward_cpu_gemm is so that we can skip the im2col if
- // we just called weight_cpu_gemm with the same input.
- void forward_cpu_gemm(const Dtype* input, const Dtype* weights,
- Dtype* output, bool skip_im2col = false);
- void forward_cpu_bias(Dtype* output, const Dtype* bias);
- void backward_cpu_gemm(const Dtype* input, const Dtype* weights,
- Dtype* output);
- void weight_cpu_gemm(const Dtype* input, const Dtype* output, Dtype*
- weights);
- void backward_cpu_bias(Dtype* bias, const Dtype* input);
-
-#ifndef CPU_ONLY
- void forward_gpu_gemm(const Dtype* col_input, const Dtype* weights,
- Dtype* output, bool skip_im2col = false);
- void forward_gpu_bias(Dtype* output, const Dtype* bias);
- void backward_gpu_gemm(const Dtype* input, const Dtype* weights,
- Dtype* col_output);
- void weight_gpu_gemm(const Dtype* col_input, const Dtype* output, Dtype*
- weights);
- void backward_gpu_bias(Dtype* bias, const Dtype* input);
-#endif
-
- /// @brief The spatial dimensions of the input.
- inline int input_shape(int i) {
- return (*bottom_shape_)[channel_axis_ + i];
- }
- // reverse_dimensions should return true iff we are implementing deconv, so
- // that conv helpers know which dimensions are which.
- virtual bool reverse_dimensions() = 0;
- // Compute height_out_ and width_out_ from other parameters.
- virtual void compute_output_shape() = 0;
-
- /// @brief The spatial dimensions of a filter kernel.
- Blob<int> kernel_shape_;
- /// @brief The spatial dimensions of the stride.
- Blob<int> stride_;
- /// @brief The spatial dimensions of the padding.
- Blob<int> pad_;
- /// @brief The spatial dimensions of the convolution input.
- Blob<int> conv_input_shape_;
- /// @brief The spatial dimensions of the col_buffer.
- vector<int> col_buffer_shape_;
- /// @brief The spatial dimensions of the output.
- vector<int> output_shape_;
- const vector<int>* bottom_shape_;
-
- int num_spatial_axes_;
- int bottom_dim_;
- int top_dim_;
-
- int channel_axis_;
- int num_;
- int channels_;
- int group_;
- int out_spatial_dim_;
- int weight_offset_;
- int num_output_;
- bool bias_term_;
- bool is_1x1_;
- bool force_nd_im2col_;
-
- private:
- // wrap im2col/col2im so we don't have to remember the (long) argument lists
- inline void conv_im2col_cpu(const Dtype* data, Dtype* col_buff) {
- if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
- im2col_cpu(data, conv_in_channels_,
- conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
- kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
- pad_.cpu_data()[0], pad_.cpu_data()[1],
- stride_.cpu_data()[0], stride_.cpu_data()[1], col_buff);
- } else {
- im2col_nd_cpu(data, num_spatial_axes_, conv_input_shape_.cpu_data(),
- col_buffer_shape_.data(), kernel_shape_.cpu_data(),
- pad_.cpu_data(), stride_.cpu_data(), col_buff);
- }
- }
- inline void conv_col2im_cpu(const Dtype* col_buff, Dtype* data) {
- if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
- col2im_cpu(col_buff, conv_in_channels_,
- conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
- kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
- pad_.cpu_data()[0], pad_.cpu_data()[1],
- stride_.cpu_data()[0], stride_.cpu_data()[1], data);
- } else {
- col2im_nd_cpu(col_buff, num_spatial_axes_, conv_input_shape_.cpu_data(),
- col_buffer_shape_.data(), kernel_shape_.cpu_data(),
- pad_.cpu_data(), stride_.cpu_data(), data);
- }
- }
-#ifndef CPU_ONLY
- inline void conv_im2col_gpu(const Dtype* data, Dtype* col_buff) {
- if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
- im2col_gpu(data, conv_in_channels_,
- conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
- kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
- pad_.cpu_data()[0], pad_.cpu_data()[1],
- stride_.cpu_data()[0], stride_.cpu_data()[1], col_buff);
- } else {
- im2col_nd_gpu(data, num_spatial_axes_, num_kernels_im2col_,
- conv_input_shape_.gpu_data(), col_buffer_.gpu_shape(),
- kernel_shape_.gpu_data(), pad_.gpu_data(),
- stride_.gpu_data(), col_buff);
- }
- }
- inline void conv_col2im_gpu(const Dtype* col_buff, Dtype* data) {
- if (!force_nd_im2col_ && num_spatial_axes_ == 2) {
- col2im_gpu(col_buff, conv_in_channels_,
- conv_input_shape_.cpu_data()[1], conv_input_shape_.cpu_data()[2],
- kernel_shape_.cpu_data()[0], kernel_shape_.cpu_data()[1],
- pad_.cpu_data()[0], pad_.cpu_data()[1],
- stride_.cpu_data()[0], stride_.cpu_data()[1], data);
- } else {
- col2im_nd_gpu(col_buff, num_spatial_axes_, num_kernels_col2im_,
- conv_input_shape_.gpu_data(), col_buffer_.gpu_shape(),
- kernel_shape_.gpu_data(), pad_.gpu_data(), stride_.gpu_data(),
- data);
- }
- }
-#endif
-
- int num_kernels_im2col_;
- int num_kernels_col2im_;
- int conv_out_channels_;
- int conv_in_channels_;
- int conv_out_spatial_dim_;
- int kernel_dim_;
- int col_offset_;
- int output_offset_;
-
- Blob<Dtype> col_buffer_;
- Blob<Dtype> bias_multiplier_;
-};
-
-/**
- * @brief Convolves the input image with a bank of learned filters,
- * and (optionally) adds biases.
- *
- * Caffe convolves by reduction to matrix multiplication. This achieves
- * high-throughput and generality of input and filter dimensions but comes at
- * the cost of memory for matrices. This makes use of efficiency in BLAS.
- *
- * The input is "im2col" transformed to a channel K' x H x W data matrix
- * for multiplication with the N x K' x H x W filter matrix to yield a
- * N' x H x W output matrix that is then "col2im" restored. K' is the
- * input channel * kernel height * kernel width dimension of the unrolled
- * inputs so that the im2col matrix has a column for each input region to
- * be filtered. col2im restores the output spatial structure by rolling up
- * the output channel N' columns of the output matrix.
- */
-template <typename Dtype>
-class ConvolutionLayer : public BaseConvolutionLayer<Dtype> {
- public:
- /**
- * @param param provides ConvolutionParameter convolution_param,
- * with ConvolutionLayer options:
- * - num_output. The number of filters.
- * - kernel_size / kernel_h / kernel_w. The filter dimensions, given by
- * kernel_size for square filters or kernel_h and kernel_w for rectangular
- * filters.
- * - stride / stride_h / stride_w (\b optional, default 1). The filter
- * stride, given by stride_size for equal dimensions or stride_h and stride_w
- * for different strides. By default the convolution is dense with stride 1.
- * - pad / pad_h / pad_w (\b optional, default 0). The zero-padding for
- * convolution, given by pad for equal dimensions or pad_h and pad_w for
- * different padding. Input padding is computed implicitly instead of
- * actually padding.
- * - group (\b optional, default 1). The number of filter groups. Group
- * convolution is a method for reducing parameterization by selectively
- * connecting input and output channels. The input and output channel dimensions must be divisible
- * by the number of groups. For group @f$ \geq 1 @f$, the
- * convolutional filters' input and output channels are separated s.t. each
- * group takes 1 / group of the input channels and makes 1 / group of the
- * output channels. Concretely 4 input channels, 8 output channels, and
- * 2 groups separate input channels 1-2 and output channels 1-4 into the
- * first group and input channels 3-4 and output channels 5-8 into the second
- * group.
- * - bias_term (\b optional, default true). Whether to have a bias.
- * - engine: convolution has CAFFE (matrix multiplication) and CUDNN (library
- * kernels + stream parallelism) engines.
- */
- explicit ConvolutionLayer(const LayerParameter& param)
- : BaseConvolutionLayer<Dtype>(param) {}
-
- virtual inline const char* type() const { return "Convolution"; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual inline bool reverse_dimensions() { return false; }
- virtual void compute_output_shape();
-};
-
-/**
- * @brief Convolve the input with a bank of learned filters, and (optionally)
- * add biases, treating filters and convolution parameters in the
- * opposite sense as ConvolutionLayer.
- *
- * ConvolutionLayer computes each output value by dotting an input window with
- * a filter; DeconvolutionLayer multiplies each input value by a filter
- * elementwise, and sums over the resulting output windows. In other words,
- * DeconvolutionLayer is ConvolutionLayer with the forward and backward passes
- * reversed. DeconvolutionLayer reuses ConvolutionParameter for its
- * parameters, but they take the opposite sense as in ConvolutionLayer (so
- * padding is removed from the output rather than added to the input, and
- * stride results in upsampling rather than downsampling).
- */
-template <typename Dtype>
-class DeconvolutionLayer : public BaseConvolutionLayer<Dtype> {
- public:
- explicit DeconvolutionLayer(const LayerParameter& param)
- : BaseConvolutionLayer<Dtype>(param) {}
-
- virtual inline const char* type() const { return "Deconvolution"; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual inline bool reverse_dimensions() { return true; }
- virtual void compute_output_shape();
-};
-
-#ifdef USE_CUDNN
-/*
- * @brief cuDNN implementation of ConvolutionLayer.
- * Fallback to ConvolutionLayer for CPU mode.
- *
- * cuDNN accelerates convolution through forward kernels for filtering and bias
- * plus backward kernels for the gradient w.r.t. the filters, biases, and
- * inputs. Caffe + cuDNN further speeds up the computation through forward
- * parallelism across groups and backward parallelism across gradients.
- *
- * The CUDNN engine does not have memory overhead for matrix buffers. For many
- * input and filter regimes the CUDNN engine is faster than the CAFFE engine,
- * but for fully-convolutional models and large inputs the CAFFE engine can be
- * faster as long as it fits in memory.
-*/
-template <typename Dtype>
-class CuDNNConvolutionLayer : public ConvolutionLayer<Dtype> {
- public:
- explicit CuDNNConvolutionLayer(const LayerParameter& param)
- : ConvolutionLayer<Dtype>(param), handles_setup_(false) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual ~CuDNNConvolutionLayer();
-
- protected:
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool handles_setup_;
- cudnnHandle_t* handle_;
- cudaStream_t* stream_;
- vector<cudnnTensorDescriptor_t> bottom_descs_, top_descs_;
- cudnnTensorDescriptor_t bias_desc_;
- cudnnFilterDescriptor_t filter_desc_;
- vector<cudnnConvolutionDescriptor_t> conv_descs_;
- int bottom_offset_, top_offset_, bias_offset_;
- size_t workspaceSizeInBytes;
- void *workspace;
-};
-#endif
-
-/**
- * @brief A helper for image operations that rearranges image regions into
- * column vectors. Used by ConvolutionLayer to perform convolution
- * by matrix multiplication.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class Im2colLayer : public Layer<Dtype> {
- public:
- explicit Im2colLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Im2col"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- /// @brief The spatial dimensions of a filter kernel.
- Blob<int> kernel_shape_;
- /// @brief The spatial dimensions of the stride.
- Blob<int> stride_;
- /// @brief The spatial dimensions of the padding.
- Blob<int> pad_;
-
- int num_spatial_axes_;
- int bottom_dim_;
- int top_dim_;
-
- int channel_axis_;
- int num_;
- int channels_;
-
- bool force_nd_im2col_;
-};
-
-// Forward declare PoolingLayer and SplitLayer for use in LRNLayer.
-template <typename Dtype> class PoolingLayer;
-template <typename Dtype> class SplitLayer;
-
-/**
- * @brief Normalize the input in a local region across or within feature maps.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class LRNLayer : public Layer<Dtype> {
- public:
- explicit LRNLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "LRN"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- virtual void CrossChannelForward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void CrossChannelForward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void WithinChannelForward(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void CrossChannelBackward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void CrossChannelBackward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void WithinChannelBackward(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int size_;
- int pre_pad_;
- Dtype alpha_;
- Dtype beta_;
- Dtype k_;
- int num_;
- int channels_;
- int height_;
- int width_;
-
- // Fields used for normalization ACROSS_CHANNELS
- // scale_ stores the intermediate summing results
- Blob<Dtype> scale_;
-
- // Fields used for normalization WITHIN_CHANNEL
- shared_ptr<SplitLayer<Dtype> > split_layer_;
- vector<Blob<Dtype>*> split_top_vec_;
- shared_ptr<PowerLayer<Dtype> > square_layer_;
- Blob<Dtype> square_input_;
- Blob<Dtype> square_output_;
- vector<Blob<Dtype>*> square_bottom_vec_;
- vector<Blob<Dtype>*> square_top_vec_;
- shared_ptr<PoolingLayer<Dtype> > pool_layer_;
- Blob<Dtype> pool_output_;
- vector<Blob<Dtype>*> pool_top_vec_;
- shared_ptr<PowerLayer<Dtype> > power_layer_;
- Blob<Dtype> power_output_;
- vector<Blob<Dtype>*> power_top_vec_;
- shared_ptr<EltwiseLayer<Dtype> > product_layer_;
- Blob<Dtype> product_input_;
- vector<Blob<Dtype>*> product_bottom_vec_;
-};
-
-
-/**
- * @brief Pools the input image by taking the max, average, etc. within regions.
- *
- * TODO(dox): thorough documentation for Forward, Backward, and proto params.
- */
-template <typename Dtype>
-class PoolingLayer : public Layer<Dtype> {
- public:
- explicit PoolingLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "Pooling"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int MinTopBlobs() const { return 1; }
- // MAX POOL layers can output an extra top blob for the mask;
- // others can only output the pooled inputs.
- virtual inline int MaxTopBlobs() const {
- return (this->layer_param_.pooling_param().pool() ==
- PoolingParameter_PoolMethod_MAX) ? 2 : 1;
- }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- int kernel_h_, kernel_w_;
- int stride_h_, stride_w_;
- int pad_h_, pad_w_;
- int channels_;
- int height_, width_;
- int pooled_height_, pooled_width_;
- bool global_pooling_;
- Blob<Dtype> rand_idx_;
- Blob<int> max_idx_;
-};
-
-#ifdef USE_CUDNN
-/*
- * @brief cuDNN implementation of PoolingLayer.
- * Fallback to PoolingLayer for CPU mode.
-*/
-template <typename Dtype>
-class CuDNNPoolingLayer : public PoolingLayer<Dtype> {
- public:
- explicit CuDNNPoolingLayer(const LayerParameter& param)
- : PoolingLayer<Dtype>(param), handles_setup_(false) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual ~CuDNNPoolingLayer();
- // Currently, cuDNN does not support the extra top blob.
- virtual inline int MinTopBlobs() const { return -1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
-
- bool handles_setup_;
- cudnnHandle_t handle_;
- cudnnTensorDescriptor_t bottom_desc_, top_desc_;
- cudnnPoolingDescriptor_t pooling_desc_;
- cudnnPoolingMode_t mode_;
-};
-#endif
-
-/**
- * @brief Does spatial pyramid pooling on the input image
- * by taking the max, average, etc. within regions
- * so that the result vector of different sized
- * images are of the same size.
- */
-template <typename Dtype>
-class SPPLayer : public Layer<Dtype> {
- public:
- explicit SPPLayer(const LayerParameter& param)
- : Layer<Dtype>(param) {}
- virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
-
- virtual inline const char* type() const { return "SPP"; }
- virtual inline int ExactNumBottomBlobs() const { return 1; }
- virtual inline int ExactNumTopBlobs() const { return 1; }
-
- protected:
- virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
- const vector<Blob<Dtype>*>& top);
- virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
- const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
- // calculates the kernel and stride dimensions for the pooling layer,
- // returns a correctly configured LayerParameter for a PoolingLayer
- virtual LayerParameter GetPoolingParam(const int pyramid_level,
- const int bottom_h, const int bottom_w, const SPPParameter spp_param);
-
- int pyramid_height_;
- int bottom_h_, bottom_w_;
- int num_;
- int channels_;
- int kernel_h_, kernel_w_;
- int pad_h_, pad_w_;
- bool reshaped_first_time_;
-
- /// the internal Split layer that feeds the pooling layers
- shared_ptr<SplitLayer<Dtype> > split_layer_;
- /// top vector holder used in call to the underlying SplitLayer::Forward
- vector<Blob<Dtype>*> split_top_vec_;
- /// bottom vector holder used in call to the underlying PoolingLayer::Forward
- vector<vector<Blob<Dtype>*>*> pooling_bottom_vecs_;
- /// the internal Pooling layers of different kernel sizes
- vector<shared_ptr<PoolingLayer<Dtype> > > pooling_layers_;
- /// top vector holders used in call to the underlying PoolingLayer::Forward
- vector<vector<Blob<Dtype>*>*> pooling_top_vecs_;
- /// pooling_outputs stores the outputs of the PoolingLayers
- vector<Blob<Dtype>*> pooling_outputs_;
- /// the internal Flatten layers that the Pooling layers feed into
- vector<FlattenLayer<Dtype>*> flatten_layers_;
- /// top vector holders used in call to the underlying FlattenLayer::Forward
- vector<vector<Blob<Dtype>*>*> flatten_top_vecs_;
- /// flatten_outputs stores the outputs of the FlattenLayers
- vector<Blob<Dtype>*> flatten_outputs_;
- /// bottom vector holder used in call to the underlying ConcatLayer::Forward
- vector<Blob<Dtype>*> concat_bottom_vec_;
- /// the internal Concat layers that the Flatten layers feed into
- shared_ptr<ConcatLayer<Dtype> > concat_layer_;
-};
-
-} // namespace caffe
-
-#endif // CAFFE_VISION_LAYERS_HPP_
diff --git a/matlab/+caffe/+test/test_io.m b/matlab/+caffe/+test/test_io.m
new file mode 100644
index 0000000..2c34bd1
--- /dev/null
+++ b/matlab/+caffe/+test/test_io.m
@@ -0,0 +1,18 @@
+classdef test_io < matlab.unittest.TestCase
+ methods (Test)
+ function test_read_write_mean(self)
+ % randomly generate mean data
+ width = 200;
+ height = 300;
+ channels = 3;
+ mean_data_write = 255 * rand(width, height, channels, 'single');
+ % write mean data to binary proto
+ mean_proto_file = tempname();
+ caffe.io.write_mean(mean_data_write, mean_proto_file);
+ % read mean data from saved binary proto and test whether they are equal
+ mean_data_read = caffe.io.read_mean(mean_proto_file);
+ self.verifyEqual(mean_data_write, mean_data_read)
+ delete(mean_proto_file);
+ end
+ end
+end
diff --git a/matlab/+caffe/io.m b/matlab/+caffe/io.m
index af8369d..4b072fe 100644
--- a/matlab/+caffe/io.m
+++ b/matlab/+caffe/io.m
@@ -29,5 +29,13 @@ classdef io
CHECK_FILE_EXIST(mean_proto_file);
mean_data = caffe_('read_mean', mean_proto_file);
end
+ function write_mean(mean_data, mean_proto_file)
+ % write_mean(mean_data, mean_proto_file)
+ % write image mean data to binaryproto file
+ % mean_data should be W x H x C with BGR channels
+ CHECK(ischar(mean_proto_file), 'mean_proto_file must be a string');
+ CHECK(isa(mean_data, 'single'), 'mean_data must be a SINGLE matrix');
+ caffe_('write_mean', mean_data, mean_proto_file);
+ end
end
end
diff --git a/matlab/+caffe/private/caffe_.cpp b/matlab/+caffe/private/caffe_.cpp
index 4e0ebc1..1641e14 100644
--- a/matlab/+caffe/private/caffe_.cpp
+++ b/matlab/+caffe/private/caffe_.cpp
@@ -188,7 +188,10 @@ static void get_solver(MEX_ARGS) {
"Usage: caffe_('get_solver', solver_file)");
char* solver_file = mxArrayToString(prhs[0]);
mxCHECK_FILE_EXIST(solver_file);
- shared_ptr<Solver<float> > solver(new caffe::SGDSolver<float>(solver_file));
+ SolverParameter solver_param;
+ ReadSolverParamsFromTextFileOrDie(solver_file, &solver_param);
+ shared_ptr<Solver<float> > solver(
+ SolverRegistry<float>::CreateSolver(solver_param));
solvers_.push_back(solver);
plhs[0] = ptr_to_handle<Solver<float> >(solver.get());
mxFree(solver_file);
@@ -478,6 +481,29 @@ static void read_mean(MEX_ARGS) {
mxFree(mean_proto_file);
}
+// Usage: caffe_('write_mean', mean_data, mean_proto_file)
+static void write_mean(MEX_ARGS) {
+ mxCHECK(nrhs == 2 && mxIsSingle(prhs[0]) && mxIsChar(prhs[1]),
+ "Usage: caffe_('write_mean', mean_data, mean_proto_file)");
+ char* mean_proto_file = mxArrayToString(prhs[1]);
+ int ndims = mxGetNumberOfDimensions(prhs[0]);
+ mxCHECK(ndims >= 2 && ndims <= 3, "mean_data must have at 2 or 3 dimensions");
+ const mwSize *dims = mxGetDimensions(prhs[0]);
+ int width = dims[0];
+ int height = dims[1];
+ int channels;
+ if (ndims == 3)
+ channels = dims[2];
+ else
+ channels = 1;
+ Blob<float> data_mean(1, channels, height, width);
+ mx_mat_to_blob(prhs[0], &data_mean, DATA);
+ BlobProto blob_proto;
+ data_mean.ToProto(&blob_proto, false);
+ WriteProtoToBinaryFile(blob_proto, mean_proto_file);
+ mxFree(mean_proto_file);
+}
+
/** -----------------------------------------------------------------
** Available commands.
**/
@@ -515,6 +541,7 @@ static handler_registry handlers[] = {
{ "get_init_key", get_init_key },
{ "reset", reset },
{ "read_mean", read_mean },
+ { "write_mean", write_mean },
// The end.
{ "END", NULL },
};
diff --git a/matlab/+caffe/run_tests.m b/matlab/+caffe/run_tests.m
index 9389685..6dbf6b2 100644
--- a/matlab/+caffe/run_tests.m
+++ b/matlab/+caffe/run_tests.m
@@ -11,7 +11,8 @@ caffe.reset_all();
% put all test cases here
results = [...
run(caffe.test.test_net) ...
- run(caffe.test.test_solver) ];
+ run(caffe.test.test_solver) ...
+ run(caffe.test.test_io) ];
% reset caffe after testing
caffe.reset_all();
diff --git a/matlab/hdf5creation/store2hdf5.m b/matlab/hdf5creation/store2hdf5.m
index 0a0016d..4e8c81d 100644
--- a/matlab/hdf5creation/store2hdf5.m
+++ b/matlab/hdf5creation/store2hdf5.m
@@ -39,8 +39,8 @@ function [curr_dat_sz, curr_lab_sz] = store2hdf5(filename, data, labels, create,
info=h5info(filename);
prev_dat_sz=info.Datasets(1).Dataspace.Size;
prev_lab_sz=info.Datasets(2).Dataspace.Size;
- assert(prev_dat_sz(1:end-1)==dat_dims(1:end-1), 'Data dimensions must match existing dimensions in dataset');
- assert(prev_lab_sz(1:end-1)==lab_dims(1:end-1), 'Label dimensions must match existing dimensions in dataset');
+ assert(all(prev_dat_sz(1:end-1)==dat_dims(1:end-1)), 'Data dimensions must match existing dimensions in dataset');
+ assert(all(prev_lab_sz(1:end-1)==lab_dims(1:end-1)), 'Label dimensions must match existing dimensions in dataset');
startloc.dat=[ones(1,length(dat_dims)-1), prev_dat_sz(end)+1];
startloc.lab=[ones(1,length(lab_dims)-1), prev_lab_sz(end)+1];
end
diff --git a/models/bvlc_reference_caffenet/train_val.prototxt b/models/bvlc_reference_caffenet/train_val.prototxt
index c79472e..e3e4279 100644
--- a/models/bvlc_reference_caffenet/train_val.prototxt
+++ b/models/bvlc_reference_caffenet/train_val.prototxt
@@ -45,7 +45,7 @@ layer {
# mean_value: 104
# mean_value: 117
# mean_value: 123
-# mirror: true
+# mirror: false
# }
data_param {
source: "examples/imagenet/ilsvrc12_val_lmdb"
diff --git a/models/finetune_flickr_style/train_val.prototxt b/models/finetune_flickr_style/train_val.prototxt
index 848a426..985353b 100644
--- a/models/finetune_flickr_style/train_val.prototxt
+++ b/models/finetune_flickr_style/train_val.prototxt
@@ -370,13 +370,6 @@ layer {
}
}
layer {
- name: "loss"
- type: "SoftmaxWithLoss"
- bottom: "fc8_flickr"
- bottom: "label"
- top: "loss"
-}
-layer {
name: "accuracy"
type: "Accuracy"
bottom: "fc8_flickr"
@@ -386,3 +379,10 @@ layer {
phase: TEST
}
}
+layer {
+ name: "loss"
+ type: "SoftmaxWithLoss"
+ bottom: "fc8_flickr"
+ bottom: "label"
+ top: "loss"
+}
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 0e2bc7e..a226414 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -1,5 +1,5 @@
if(NOT HAVE_PYTHON)
- message(STATUS "Python interface is disabled or not all required dependecies found. Building without it...")
+ message(STATUS "Python interface is disabled or not all required dependencies found. Building without it...")
return()
endif()
diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp
index ccd5776..69d5533 100644
--- a/python/caffe/_caffe.cpp
+++ b/python/caffe/_caffe.cpp
@@ -15,7 +15,9 @@
#include <fstream> // NOLINT
#include "caffe/caffe.hpp"
-#include "caffe/python_layer.hpp"
+#include "caffe/layers/memory_data_layer.hpp"
+#include "caffe/layers/python_layer.hpp"
+#include "caffe/sgd_solvers.hpp"
// Temporary solution for numpy < 1.7 versions: old macro, no promises.
// You're strongly advised to upgrade to >= 1.7.
@@ -133,8 +135,8 @@ void Net_SetInputArrays(Net<Dtype>* net, bp::object data_obj,
Solver<Dtype>* GetSolverFromFile(const string& filename) {
SolverParameter param;
- ReadProtoFromTextFileOrDie(filename, ¶m);
- return GetSolver<Dtype>(param);
+ ReadSolverParamsFromTextFileOrDie(filename, ¶m);
+ return SolverRegistry<Dtype>::CreateSolver(param);
}
struct NdarrayConverterGenerator {
@@ -286,7 +288,8 @@ BOOST_PYTHON_MODULE(_caffe) {
.def("solve", static_cast<void (Solver<Dtype>::*)(const char*)>(
&Solver<Dtype>::Solve), SolveOverloads())
.def("step", &Solver<Dtype>::Step)
- .def("restore", &Solver<Dtype>::Restore);
+ .def("restore", &Solver<Dtype>::Restore)
+ .def("snapshot", &Solver<Dtype>::Snapshot);
bp::class_<SGDSolver<Dtype>, bp::bases<Solver<Dtype> >,
shared_ptr<SGDSolver<Dtype> >, boost::noncopyable>(
diff --git a/python/caffe/draw.py b/python/caffe/draw.py
index a002b60..f8bf572 100644
--- a/python/caffe/draw.py
+++ b/python/caffe/draw.py
@@ -82,11 +82,11 @@ def get_layer_label(layer, rankdir):
separator,
layer.type,
separator,
- layer.convolution_param.kernel_size,
+ layer.convolution_param.kernel_size[0] if len(layer.convolution_param.kernel_size._values) else 1,
separator,
- layer.convolution_param.stride,
+ layer.convolution_param.stride[0] if len(layer.convolution_param.stride._values) else 1,
separator,
- layer.convolution_param.pad)
+ layer.convolution_param.pad[0] if len(layer.convolution_param.pad._values) else 0)
elif layer.type == 'Pooling':
pooling_types_dict = get_pooling_types_dict()
node_label = '"%s%s(%s %s)%skernel size: %d%sstride: %d%spad: %d"' %\
diff --git a/python/caffe/io.py b/python/caffe/io.py
index 0cad721..14942be 100644
--- a/python/caffe/io.py
+++ b/python/caffe/io.py
@@ -20,23 +20,26 @@ def blobproto_to_array(blob, return_diff=False):
Convert a blob proto to an array. In default, we will just return the data,
unless return_diff is True, in which case we will return the diff.
"""
+ # Read the data into an array
if return_diff:
- return np.array(blob.diff).reshape(
- blob.num, blob.channels, blob.height, blob.width)
+ data = np.array(blob.diff)
else:
- return np.array(blob.data).reshape(
- blob.num, blob.channels, blob.height, blob.width)
+ data = np.array(blob.data)
+ # Reshape the array
+ if blob.HasField('num') or blob.HasField('channels') or blob.HasField('height') or blob.HasField('width'):
+ # Use legacy 4D shape
+ return data.reshape(blob.num, blob.channels, blob.height, blob.width)
+ else:
+ return data.reshape(blob.shape.dim)
def array_to_blobproto(arr, diff=None):
- """Converts a 4-dimensional array to blob proto. If diff is given, also
+ """Converts a N-dimensional array to blob proto. If diff is given, also
convert the diff. You need to make sure that arr and diff have the same
shape, and this function does not do sanity check.
"""
- if arr.ndim != 4:
- raise ValueError('Incorrect array shape.')
blob = caffe_pb2.BlobProto()
- blob.num, blob.channels, blob.height, blob.width = arr.shape
+ blob.shape.dim.extend(arr.shape)
blob.data.extend(arr.astype(float).flat)
if diff is not None:
blob.diff.extend(diff.astype(float).flat)
@@ -175,9 +178,9 @@ class Transformer:
if raw_scale is not None:
decaf_in /= raw_scale
if channel_swap is not None:
- decaf_in = decaf_in[channel_swap, :, :]
+ decaf_in = decaf_in[np.argsort(channel_swap), :, :]
if transpose is not None:
- decaf_in = decaf_in.transpose([transpose[t] for t in transpose])
+ decaf_in = decaf_in.transpose(np.argsort(transpose))
return decaf_in
def set_transpose(self, in_, order):
diff --git a/python/caffe/pycaffe.py b/python/caffe/pycaffe.py
index 8ea24da..31dc702 100644
--- a/python/caffe/pycaffe.py
+++ b/python/caffe/pycaffe.py
@@ -146,8 +146,6 @@ def _Net_backward(self, diffs=None, start=None, end=None, **kwargs):
# Set top diffs according to defined shapes and make arrays single and
# C-contiguous as Caffe expects.
for top, diff in kwargs.iteritems():
- if diff.ndim != 4:
- raise Exception('{} diff is not 4-d'.format(top))
if diff.shape[0] != self.blobs[top].num:
raise Exception('Diff is not batch sized')
self.blobs[top].diff[...] = diff
@@ -218,9 +216,9 @@ def _Net_forward_backward_all(self, blobs=None, diffs=None, **kwargs):
batch_blobs = self.forward(blobs=blobs, **fb)
batch_diffs = self.backward(diffs=diffs, **bb)
for out, out_blobs in batch_blobs.iteritems():
- all_outs[out].extend(out_blobs)
+ all_outs[out].extend(out_blobs.copy())
for diff, out_diffs in batch_diffs.iteritems():
- all_diffs[diff].extend(out_diffs)
+ all_diffs[diff].extend(out_diffs.copy())
# Package in ndarray.
for out, diff in zip(all_outs, all_diffs):
all_outs[out] = np.asarray(all_outs[out])
diff --git a/python/caffe/test/test_io.py b/python/caffe/test/test_io.py
new file mode 100644
index 0000000..8c86ef7
--- /dev/null
+++ b/python/caffe/test/test_io.py
@@ -0,0 +1,41 @@
+import numpy as np
+import unittest
+
+import caffe
+
+class TestBlobProtoToArray(unittest.TestCase):
+
+ def test_old_format(self):
+ data = np.zeros((10,10))
+ blob = caffe.proto.caffe_pb2.BlobProto()
+ blob.data.extend(list(data.flatten()))
+ shape = (1,1,10,10)
+ blob.num, blob.channels, blob.height, blob.width = shape
+
+ arr = caffe.io.blobproto_to_array(blob)
+ self.assertEqual(arr.shape, shape)
+
+ def test_new_format(self):
+ data = np.zeros((10,10))
+ blob = caffe.proto.caffe_pb2.BlobProto()
+ blob.data.extend(list(data.flatten()))
+ blob.shape.dim.extend(list(data.shape))
+
+ arr = caffe.io.blobproto_to_array(blob)
+ self.assertEqual(arr.shape, data.shape)
+
+ def test_no_shape(self):
+ data = np.zeros((10,10))
+ blob = caffe.proto.caffe_pb2.BlobProto()
+ blob.data.extend(list(data.flatten()))
+
+ with self.assertRaises(ValueError):
+ caffe.io.blobproto_to_array(blob)
+
+ def test_scalar(self):
+ data = np.ones((1)) * 123
+ blob = caffe.proto.caffe_pb2.BlobProto()
+ blob.data.extend(list(data.flatten()))
+
+ arr = caffe.io.blobproto_to_array(blob)
+ self.assertEqual(arr, 123)
diff --git a/python/caffe/test/test_python_layer.py b/python/caffe/test/test_python_layer.py
index 8ed8665..e46b711 100644
--- a/python/caffe/test/test_python_layer.py
+++ b/python/caffe/test/test_python_layer.py
@@ -77,6 +77,8 @@ def parameter_net_file():
return f.name
+ at unittest.skipIf('Python' not in caffe.layer_type_list(),
+ 'Caffe built without Python layer support')
class TestPythonLayer(unittest.TestCase):
def setUp(self):
net_file = python_net_file()
diff --git a/python/caffe/test/test_python_layer_with_param_str.py b/python/caffe/test/test_python_layer_with_param_str.py
index 3d0f107..c36048a 100644
--- a/python/caffe/test/test_python_layer_with_param_str.py
+++ b/python/caffe/test/test_python_layer_with_param_str.py
@@ -38,6 +38,8 @@ def python_param_net_file():
return f.name
+ at unittest.skipIf('Python' not in caffe.layer_type_list(),
+ 'Caffe built without Python layer support')
class TestLayerWithParam(unittest.TestCase):
def setUp(self):
net_file = python_param_net_file()
diff --git a/python/caffe/test/test_solver.py b/python/caffe/test/test_solver.py
index 9cfc10d..f618fde 100644
--- a/python/caffe/test/test_solver.py
+++ b/python/caffe/test/test_solver.py
@@ -16,7 +16,8 @@ class TestSolver(unittest.TestCase):
f.write("""net: '""" + net_f + """'
test_iter: 10 test_interval: 10 base_lr: 0.01 momentum: 0.9
weight_decay: 0.0005 lr_policy: 'inv' gamma: 0.0001 power: 0.75
- display: 100 max_iter: 100 snapshot_after_train: false""")
+ display: 100 max_iter: 100 snapshot_after_train: false
+ snapshot_prefix: "model" """)
f.close()
self.solver = caffe.SGDSolver(f.name)
# also make sure get_solver runs
@@ -51,3 +52,11 @@ class TestSolver(unittest.TestCase):
total += p.data.sum() + p.diff.sum()
for bl in six.itervalues(net.blobs):
total += bl.data.sum() + bl.diff.sum()
+
+ def test_snapshot(self):
+ self.solver.snapshot()
+ # Check that these files exist and then remove them
+ files = ['model_iter_0.caffemodel', 'model_iter_0.solverstate']
+ for fn in files:
+ assert os.path.isfile(fn)
+ os.remove(fn)
diff --git a/python/detect.py b/python/detect.py
index 691098f..1aba964 100755
--- a/python/detect.py
+++ b/python/detect.py
@@ -46,7 +46,7 @@ def main(argv):
parser.add_argument(
"--model_def",
default=os.path.join(pycaffe_dir,
- "../models/bvlc_reference_caffenet/deploy.prototxt.prototxt"),
+ "../models/bvlc_reference_caffenet/deploy.prototxt"),
help="Model definition file."
)
parser.add_argument(
diff --git a/scripts/download_model_binary.py b/scripts/download_model_binary.py
index 03a50f6..66f72f2 100755
--- a/scripts/download_model_binary.py
+++ b/scripts/download_model_binary.py
@@ -18,7 +18,7 @@ def reporthook(count, block_size, total_size):
if count == 0:
start_time = time.time()
return
- duration = time.time() - start_time
+ duration = (time.time() - start_time) or 0.01
progress_size = int(count * block_size)
speed = int(progress_size / (1024 * duration))
percent = int(count * block_size * 100 / total_size)
diff --git a/scripts/travis/travis_install.sh b/scripts/travis/travis_install.sh
index d6c6e22..d18dc22 100755
--- a/scripts/travis/travis_install.sh
+++ b/scripts/travis/travis_install.sh
@@ -23,7 +23,7 @@ apt-get install \
# Caffe requires a minimum CMake version of 2.8.8.
if $WITH_CMAKE; then
# cmake 3 will make sure that the python interpreter and libraries match
- wget http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh -O cmake3.sh
+ wget --no-check-certificate http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh -O cmake3.sh
chmod +x cmake3.sh
./cmake3.sh --prefix=/usr/ --skip-license --exclude-subdir
fi
@@ -61,35 +61,39 @@ rm -f $LMDB_FILE
# than using pip for everything).
export PATH=$CONDA_DIR/bin:$PATH
if [ ! -d $CONDA_DIR ]; then
- if [ "$PYTHON_VERSION" -eq "3" ]; then
- wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
- else
- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
- fi
- chmod +x miniconda.sh
- ./miniconda.sh -b -p $CONDA_DIR
-
- conda update --yes conda
- conda install --yes numpy scipy matplotlib scikit-image pip
- # Let conda install boost (so that boost_python matches)
- conda install --yes -c https://conda.binstar.org/menpo boost=1.56.0
+ if [ "$PYTHON_VERSION" -eq "3" ]; then
+ wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+ else
+ wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
+ fi
+ chmod +x miniconda.sh
+ ./miniconda.sh -b -p $CONDA_DIR
+
+ conda update --yes conda
+ # The version of boost we're using for Python 3 depends on 3.4 for now.
+ if [ "$PYTHON_VERSION" -eq "3" ]; then
+ conda install --yes python=3.4
+ fi
+ conda install --yes numpy scipy matplotlib scikit-image pip
+ # Let conda install boost (so that boost_python matches)
+ conda install --yes -c https://conda.binstar.org/menpo boost=1.56.0
fi
# install protobuf 3 (just use the miniconda3 directory to avoid having to setup the path again)
if [ "$PYTHON_VERSION" -eq "3" ] && [ ! -e "$CONDA_DIR/bin/protoc" ]; then
- pushd .
- wget https://github.com/google/protobuf/archive/v3.0.0-alpha-3.1.tar.gz -O protobuf-3.tar.gz
- tar -C /tmp -xzvf protobuf-3.tar.gz
- cd /tmp/protobuf-3*/
- ./autogen.sh
- ./configure --prefix=$CONDA_DIR
- $MAKE
- $MAKE install
- popd
+ pushd .
+ wget https://github.com/google/protobuf/archive/v3.0.0-alpha-3.1.tar.gz -O protobuf-3.tar.gz
+ tar -C /tmp -xzvf protobuf-3.tar.gz
+ cd /tmp/protobuf-3*/
+ ./autogen.sh
+ ./configure --prefix=$CONDA_DIR
+ $MAKE
+ $MAKE install
+ popd
fi
if [ "$PYTHON_VERSION" -eq "3" ]; then
- pip install --pre protobuf
+ pip install --pre protobuf
else
- pip install protobuf
+ pip install protobuf
fi
diff --git a/src/caffe/data_reader.cpp b/src/caffe/data_reader.cpp
index 1637820..9f019bb 100644
--- a/src/caffe/data_reader.cpp
+++ b/src/caffe/data_reader.cpp
@@ -4,8 +4,8 @@
#include <vector>
#include "caffe/common.hpp"
-#include "caffe/data_layers.hpp"
#include "caffe/data_reader.hpp"
+#include "caffe/layers/data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
namespace caffe {
diff --git a/src/caffe/layer_factory.cpp b/src/caffe/layer_factory.cpp
index 926c7d8..76d851a 100644
--- a/src/caffe/layer_factory.cpp
+++ b/src/caffe/layer_factory.cpp
@@ -7,11 +7,28 @@
#include "caffe/layer.hpp"
#include "caffe/layer_factory.hpp"
+#include "caffe/layers/conv_layer.hpp"
+#include "caffe/layers/lrn_layer.hpp"
+#include "caffe/layers/pooling_layer.hpp"
+#include "caffe/layers/relu_layer.hpp"
+#include "caffe/layers/sigmoid_layer.hpp"
+#include "caffe/layers/softmax_layer.hpp"
+#include "caffe/layers/tanh_layer.hpp"
#include "caffe/proto/caffe.pb.h"
-#include "caffe/vision_layers.hpp"
+
+#ifdef USE_CUDNN
+#include "caffe/layers/cudnn_conv_layer.hpp"
+#include "caffe/layers/cudnn_lcn_layer.hpp"
+#include "caffe/layers/cudnn_lrn_layer.hpp"
+#include "caffe/layers/cudnn_pooling_layer.hpp"
+#include "caffe/layers/cudnn_relu_layer.hpp"
+#include "caffe/layers/cudnn_sigmoid_layer.hpp"
+#include "caffe/layers/cudnn_softmax_layer.hpp"
+#include "caffe/layers/cudnn_tanh_layer.hpp"
+#endif
#ifdef WITH_PYTHON_LAYER
-#include "caffe/python_layer.hpp"
+#include "caffe/layers/python_layer.hpp"
#endif
namespace caffe {
@@ -54,10 +71,8 @@ shared_ptr<Layer<Dtype> > GetPoolingLayer(const LayerParameter& param) {
return shared_ptr<Layer<Dtype> >(new PoolingLayer<Dtype>(param));
#ifdef USE_CUDNN
} else if (engine == PoolingParameter_Engine_CUDNN) {
- PoolingParameter p_param = param.pooling_param();
- if (p_param.pad() || p_param.pad_h() || p_param.pad_w() ||
- param.top_size() > 1) {
- LOG(INFO) << "CUDNN does not support padding or multiple tops. "
+ if (param.top_size() > 1) {
+ LOG(INFO) << "cuDNN does not support multiple tops. "
<< "Using Caffe's own pooling layer.";
return shared_ptr<Layer<Dtype> >(new PoolingLayer<Dtype>(param));
}
@@ -70,6 +85,43 @@ shared_ptr<Layer<Dtype> > GetPoolingLayer(const LayerParameter& param) {
REGISTER_LAYER_CREATOR(Pooling, GetPoolingLayer);
+// Get LRN layer according to engine
+template <typename Dtype>
+shared_ptr<Layer<Dtype> > GetLRNLayer(const LayerParameter& param) {
+ LRNParameter_Engine engine = param.lrn_param().engine();
+
+ if (engine == LRNParameter_Engine_DEFAULT) {
+#ifdef USE_CUDNN
+ engine = LRNParameter_Engine_CUDNN;
+#else
+ engine = LRNParameter_Engine_CAFFE;
+#endif
+ }
+
+ if (engine == LRNParameter_Engine_CAFFE) {
+ return shared_ptr<Layer<Dtype> >(new LRNLayer<Dtype>(param));
+#ifdef USE_CUDNN
+ } else if (engine == LRNParameter_Engine_CUDNN) {
+ LRNParameter lrn_param = param.lrn_param();
+
+ if (lrn_param.norm_region() ==LRNParameter_NormRegion_WITHIN_CHANNEL) {
+ return shared_ptr<Layer<Dtype> >(new CuDNNLCNLayer<Dtype>(param));
+ } else {
+ // local size is too big to be handled through cuDNN
+ if (param.lrn_param().local_size() > CUDNN_LRN_MAX_N) {
+ return shared_ptr<Layer<Dtype> >(new LRNLayer<Dtype>(param));
+ } else {
+ return shared_ptr<Layer<Dtype> >(new CuDNNLRNLayer<Dtype>(param));
+ }
+ }
+#endif
+ } else {
+ LOG(FATAL) << "Layer " << param.name() << " has unknown engine.";
+ }
+}
+
+REGISTER_LAYER_CREATOR(LRN, GetLRNLayer);
+
// Get relu layer according to engine.
template <typename Dtype>
shared_ptr<Layer<Dtype> > GetReLULayer(const LayerParameter& param) {
diff --git a/src/caffe/layers/absval_layer.cpp b/src/caffe/layers/absval_layer.cpp
index 5ce28c9..855bf0b 100644
--- a/src/caffe/layers/absval_layer.cpp
+++ b/src/caffe/layers/absval_layer.cpp
@@ -1,7 +1,6 @@
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/neuron_layers.hpp"
+#include "caffe/layers/absval_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
diff --git a/src/caffe/layers/absval_layer.cu b/src/caffe/layers/absval_layer.cu
index bb310e1..6c927e6 100644
--- a/src/caffe/layers/absval_layer.cu
+++ b/src/caffe/layers/absval_layer.cu
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/absval_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/accuracy_layer.cpp b/src/caffe/layers/accuracy_layer.cpp
index e2d8d9f..4eddbb5 100644
--- a/src/caffe/layers/accuracy_layer.cpp
+++ b/src/caffe/layers/accuracy_layer.cpp
@@ -1,12 +1,9 @@
-#include <algorithm>
#include <functional>
#include <utility>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/accuracy_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/argmax_layer.cpp b/src/caffe/layers/argmax_layer.cpp
index c4040cd..2d3d6f2 100644
--- a/src/caffe/layers/argmax_layer.cpp
+++ b/src/caffe/layers/argmax_layer.cpp
@@ -3,31 +3,52 @@
#include <utility>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/argmax_layer.hpp"
namespace caffe {
template <typename Dtype>
void ArgMaxLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
- out_max_val_ = this->layer_param_.argmax_param().out_max_val();
- top_k_ = this->layer_param_.argmax_param().top_k();
- CHECK_GE(top_k_, 1) << " top k must not be less than 1.";
- CHECK_LE(top_k_, bottom[0]->count() / bottom[0]->num())
- << "top_k must be less than or equal to the number of classes.";
+ const ArgMaxParameter& argmax_param = this->layer_param_.argmax_param();
+ out_max_val_ = argmax_param.out_max_val();
+ top_k_ = argmax_param.top_k();
+ has_axis_ = argmax_param.has_axis();
+ CHECK_GE(top_k_, 1) << "top k must not be less than 1.";
+ if (has_axis_) {
+ axis_ = bottom[0]->CanonicalAxisIndex(argmax_param.axis());
+ CHECK_GE(axis_, 0) << "axis must not be less than 0.";
+ CHECK_LE(axis_, bottom[0]->num_axes()) <<
+ "axis must be less than or equal to the number of axis.";
+ CHECK_LE(top_k_, bottom[0]->shape(axis_))
+ << "top_k must be less than or equal to the dimension of the axis.";
+ } else {
+ CHECK_LE(top_k_, bottom[0]->count(1))
+ << "top_k must be less than or equal to"
+ " the dimension of the flattened bottom blob per instance.";
+ }
}
template <typename Dtype>
void ArgMaxLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
- if (out_max_val_) {
- // Produces max_ind and max_val
- top[0]->Reshape(bottom[0]->num(), 2, top_k_, 1);
+ int num_top_axes = bottom[0]->num_axes();
+ if ( num_top_axes < 3 ) num_top_axes = 3;
+ std::vector<int> shape(num_top_axes, 1);
+ if (has_axis_) {
+ // Produces max_ind or max_val per axis
+ shape = bottom[0]->shape();
+ shape[axis_] = top_k_;
} else {
- // Produces only max_ind
- top[0]->Reshape(bottom[0]->num(), 1, top_k_, 1);
+ shape[0] = bottom[0]->shape(0);
+ // Produces max_ind
+ shape[2] = top_k_;
+ if (out_max_val_) {
+ // Produces max_ind and max_val
+ shape[1] = 2;
+ }
}
+ top[0]->Reshape(shape);
}
template <typename Dtype>
@@ -35,23 +56,40 @@ void ArgMaxLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const Dtype* bottom_data = bottom[0]->cpu_data();
Dtype* top_data = top[0]->mutable_cpu_data();
- int num = bottom[0]->num();
- int dim = bottom[0]->count() / bottom[0]->num();
+ int dim, axis_dist;
+ if (has_axis_) {
+ dim = bottom[0]->shape(axis_);
+ // Distance between values of axis in blob
+ axis_dist = bottom[0]->count(axis_) / dim;
+ } else {
+ dim = bottom[0]->count(1);
+ axis_dist = 1;
+ }
+ int num = bottom[0]->count() / dim;
+ std::vector<std::pair<Dtype, int> > bottom_data_vector(dim);
for (int i = 0; i < num; ++i) {
- std::vector<std::pair<Dtype, int> > bottom_data_vector;
for (int j = 0; j < dim; ++j) {
- bottom_data_vector.push_back(
- std::make_pair(bottom_data[i * dim + j], j));
+ bottom_data_vector[j] = std::make_pair(
+ bottom_data[(i / axis_dist * dim + j) * axis_dist + i % axis_dist], j);
}
std::partial_sort(
bottom_data_vector.begin(), bottom_data_vector.begin() + top_k_,
bottom_data_vector.end(), std::greater<std::pair<Dtype, int> >());
for (int j = 0; j < top_k_; ++j) {
- top_data[top[0]->offset(i, 0, j)] = bottom_data_vector[j].second;
- }
- if (out_max_val_) {
- for (int j = 0; j < top_k_; ++j) {
- top_data[top[0]->offset(i, 1, j)] = bottom_data_vector[j].first;
+ if (out_max_val_) {
+ if (has_axis_) {
+ // Produces max_val per axis
+ top_data[(i / axis_dist * top_k_ + j) * axis_dist + i % axis_dist]
+ = bottom_data_vector[j].first;
+ } else {
+ // Produces max_ind and max_val
+ top_data[2 * i * top_k_ + j] = bottom_data_vector[j].second;
+ top_data[2 * i * top_k_ + top_k_ + j] = bottom_data_vector[j].first;
+ }
+ } else {
+ // Produces max_ind per axis
+ top_data[(i / axis_dist * top_k_ + j) * axis_dist + i % axis_dist]
+ = bottom_data_vector[j].second;
}
}
}
diff --git a/src/caffe/layers/base_conv_layer.cpp b/src/caffe/layers/base_conv_layer.cpp
index c6b4755..f6f14cd 100644
--- a/src/caffe/layers/base_conv_layer.cpp
+++ b/src/caffe/layers/base_conv_layer.cpp
@@ -2,10 +2,9 @@
#include <vector>
#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/base_conv_layer.hpp"
#include "caffe/util/im2col.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/base_data_layer.cpp b/src/caffe/layers/base_data_layer.cpp
index b90bd4e..989319f 100644
--- a/src/caffe/layers/base_data_layer.cpp
+++ b/src/caffe/layers/base_data_layer.cpp
@@ -1,10 +1,13 @@
#include <boost/thread.hpp>
-#include <string>
#include <vector>
-#include "caffe/data_layers.hpp"
-#include "caffe/net.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/blob.hpp"
+#include "caffe/data_transformer.hpp"
+#include "caffe/internal_thread.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/layers/base_data_layer.hpp"
+#include "caffe/proto/caffe.pb.h"
+#include "caffe/util/blocking_queue.hpp"
namespace caffe {
diff --git a/src/caffe/layers/base_data_layer.cu b/src/caffe/layers/base_data_layer.cu
index ff6e412..4056d36 100644
--- a/src/caffe/layers/base_data_layer.cu
+++ b/src/caffe/layers/base_data_layer.cu
@@ -1,6 +1,6 @@
#include <vector>
-#include "caffe/data_layers.hpp"
+#include "caffe/layers/base_data_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/batch_norm_layer.cpp b/src/caffe/layers/batch_norm_layer.cpp
new file mode 100644
index 0000000..a69d8f9
--- /dev/null
+++ b/src/caffe/layers/batch_norm_layer.cpp
@@ -0,0 +1,239 @@
+#include <algorithm>
+#include <vector>
+
+#include "caffe/layers/batch_norm_layer.hpp"
+#include "caffe/util/math_functions.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void BatchNormLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ BatchNormParameter param = this->layer_param_.batch_norm_param();
+ moving_average_fraction_ = param.moving_average_fraction();
+ use_global_stats_ = this->phase_ == TEST;
+ if (param.has_use_global_stats())
+ use_global_stats_ = param.use_global_stats();
+ if (bottom[0]->num_axes() == 1)
+ channels_ = 1;
+ else
+ channels_ = bottom[0]->shape(1);
+ eps_ = param.eps();
+ if (this->blobs_.size() > 0) {
+ LOG(INFO) << "Skipping parameter initialization";
+ } else {
+ this->blobs_.resize(3);
+ vector<int> sz;
+ sz.push_back(channels_);
+ this->blobs_[0].reset(new Blob<Dtype>(sz));
+ this->blobs_[1].reset(new Blob<Dtype>(sz));
+ sz[0]=1;
+ this->blobs_[2].reset(new Blob<Dtype>(sz));
+ for (int i = 0; i < 3; ++i) {
+ caffe_set(this->blobs_[i]->count(), Dtype(0),
+ this->blobs_[i]->mutable_cpu_data());
+ }
+ }
+}
+
+template <typename Dtype>
+void BatchNormLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ if (bottom[0]->num_axes() >= 1)
+ CHECK_EQ(bottom[0]->shape(1), channels_);
+ top[0]->ReshapeLike(*bottom[0]);
+
+ vector<int> sz;
+ sz.push_back(channels_);
+ mean_.Reshape(sz);
+ variance_.Reshape(sz);
+ temp_.ReshapeLike(*bottom[0]);
+ x_norm_.ReshapeLike(*bottom[0]);
+ sz[0]=bottom[0]->shape(0);
+ batch_sum_multiplier_.Reshape(sz);
+
+ int spatial_dim = bottom[0]->count()/(channels_*bottom[0]->shape(0));
+ if (spatial_sum_multiplier_.num_axes() == 0 ||
+ spatial_sum_multiplier_.shape(0) != spatial_dim) {
+ sz[0] = spatial_dim;
+ spatial_sum_multiplier_.Reshape(sz);
+ Dtype* multiplier_data = spatial_sum_multiplier_.mutable_cpu_data();
+ caffe_set(spatial_sum_multiplier_.count(), Dtype(1), multiplier_data);
+ }
+
+ int numbychans = channels_*bottom[0]->shape(0);
+ if (num_by_chans_.num_axes() == 0 ||
+ num_by_chans_.shape(0) != numbychans) {
+ sz[0] = numbychans;
+ num_by_chans_.Reshape(sz);
+ caffe_set(batch_sum_multiplier_.count(), Dtype(1),
+ batch_sum_multiplier_.mutable_cpu_data());
+ }
+}
+
+template <typename Dtype>
+void BatchNormLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ const Dtype* bottom_data = bottom[0]->cpu_data();
+ Dtype* top_data = top[0]->mutable_cpu_data();
+ int num = bottom[0]->shape(0);
+ int spatial_dim = bottom[0]->count()/(bottom[0]->shape(0)*channels_);
+
+ if (bottom[0] != top[0]) {
+ caffe_copy(bottom[0]->count(), bottom_data, top_data);
+ }
+
+ if (use_global_stats_) {
+ // use the stored mean/variance estimates.
+ const Dtype scale_factor = this->blobs_[2]->cpu_data()[0] == 0 ?
+ 0 : 1 / this->blobs_[2]->cpu_data()[0];
+ caffe_cpu_scale(variance_.count(), scale_factor,
+ this->blobs_[0]->cpu_data(), mean_.mutable_cpu_data());
+ caffe_cpu_scale(variance_.count(), scale_factor,
+ this->blobs_[1]->cpu_data(), variance_.mutable_cpu_data());
+ } else {
+ // compute mean
+ caffe_cpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim,
+ 1. / (num * spatial_dim), bottom_data,
+ spatial_sum_multiplier_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.cpu_data(), batch_sum_multiplier_.cpu_data(), 0.,
+ mean_.mutable_cpu_data());
+ }
+
+ // subtract mean
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.cpu_data(), mean_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
+ spatial_dim, 1, -1, num_by_chans_.cpu_data(),
+ spatial_sum_multiplier_.cpu_data(), 1., top_data);
+
+ if (!use_global_stats_) {
+ // compute variance using var(X) = E((X-EX)^2)
+ caffe_powx(top[0]->count(), top_data, Dtype(2),
+ temp_.mutable_cpu_data()); // (X-EX)^2
+ caffe_cpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim,
+ 1. / (num * spatial_dim), temp_.cpu_data(),
+ spatial_sum_multiplier_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.cpu_data(), batch_sum_multiplier_.cpu_data(), 0.,
+ variance_.mutable_cpu_data()); // E((X_EX)^2)
+
+ // compute and save moving average
+ this->blobs_[2]->mutable_cpu_data()[0] *= moving_average_fraction_;
+ this->blobs_[2]->mutable_cpu_data()[0] += 1;
+ caffe_cpu_axpby(mean_.count(), Dtype(1), mean_.cpu_data(),
+ moving_average_fraction_, this->blobs_[0]->mutable_cpu_data());
+ int m = bottom[0]->count()/channels_;
+ Dtype bias_correction_factor = m > 1 ? Dtype(m)/(m-1) : 1;
+ caffe_cpu_axpby(variance_.count(), bias_correction_factor,
+ variance_.cpu_data(), moving_average_fraction_,
+ this->blobs_[1]->mutable_cpu_data());
+ }
+
+ // normalize variance
+ caffe_add_scalar(variance_.count(), eps_, variance_.mutable_cpu_data());
+ caffe_powx(variance_.count(), variance_.cpu_data(), Dtype(0.5),
+ variance_.mutable_cpu_data());
+
+ // replicate variance to input size
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.cpu_data(), variance_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
+ spatial_dim, 1, 1., num_by_chans_.cpu_data(),
+ spatial_sum_multiplier_.cpu_data(), 0., temp_.mutable_cpu_data());
+ caffe_div(temp_.count(), top_data, temp_.cpu_data(), top_data);
+ // TODO(cdoersch): The caching is only needed because later in-place layers
+ // might clobber the data. Can we skip this if they won't?
+ caffe_copy(x_norm_.count(), top_data,
+ x_norm_.mutable_cpu_data());
+}
+
+template <typename Dtype>
+void BatchNormLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down,
+ const vector<Blob<Dtype>*>& bottom) {
+ const Dtype* top_diff;
+ if (bottom[0] != top[0]) {
+ top_diff = top[0]->cpu_diff();
+ } else {
+ caffe_copy(x_norm_.count(), top[0]->cpu_diff(), x_norm_.mutable_cpu_diff());
+ top_diff = x_norm_.cpu_diff();
+ }
+ Dtype* bottom_diff = bottom[0]->mutable_cpu_diff();
+ if (use_global_stats_) {
+ caffe_div(temp_.count(), top_diff, temp_.cpu_data(), bottom_diff);
+ return;
+ }
+ const Dtype* top_data = x_norm_.cpu_data();
+ int num = bottom[0]->shape()[0];
+ int spatial_dim = bottom[0]->count()/(bottom[0]->shape(0)*channels_);
+ // if Y = (X-mean(X))/(sqrt(var(X)+eps)), then
+ //
+ // dE(Y)/dX =
+ // (dE/dY - mean(dE/dY) - mean(dE/dY \cdot Y) \cdot Y)
+ // ./ sqrt(var(X) + eps)
+ //
+ // where \cdot and ./ are hadamard product and elementwise division,
+ // respectively, dE/dY is the top diff, and mean/var/sum are all computed
+ // along all dimensions except the channels dimension. In the above
+ // equation, the operations allow for expansion (i.e. broadcast) along all
+ // dimensions except the channels dimension where required.
+
+ // sum(dE/dY \cdot Y)
+ caffe_mul(temp_.count(), top_data, top_diff, bottom_diff);
+ caffe_cpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim, 1.,
+ bottom_diff, spatial_sum_multiplier_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.cpu_data(), batch_sum_multiplier_.cpu_data(), 0.,
+ mean_.mutable_cpu_data());
+
+ // reshape (broadcast) the above
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.cpu_data(), mean_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
+ spatial_dim, 1, 1., num_by_chans_.cpu_data(),
+ spatial_sum_multiplier_.cpu_data(), 0., bottom_diff);
+
+ // sum(dE/dY \cdot Y) \cdot Y
+ caffe_mul(temp_.count(), top_data, bottom_diff, bottom_diff);
+
+ // sum(dE/dY)-sum(dE/dY \cdot Y) \cdot Y
+ caffe_cpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim, 1.,
+ top_diff, spatial_sum_multiplier_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.cpu_data(), batch_sum_multiplier_.cpu_data(), 0.,
+ mean_.mutable_cpu_data());
+ // reshape (broadcast) the above to make
+ // sum(dE/dY)-sum(dE/dY \cdot Y) \cdot Y
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.cpu_data(), mean_.cpu_data(), 0.,
+ num_by_chans_.mutable_cpu_data());
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num * channels_,
+ spatial_dim, 1, 1., num_by_chans_.cpu_data(),
+ spatial_sum_multiplier_.cpu_data(), 1., bottom_diff);
+
+ // dE/dY - mean(dE/dY)-mean(dE/dY \cdot Y) \cdot Y
+ caffe_cpu_axpby(temp_.count(), Dtype(1), top_diff,
+ Dtype(-1. / (num * spatial_dim)), bottom_diff);
+
+ // note: temp_ still contains sqrt(var(X)+eps), computed during the forward
+ // pass.
+ caffe_div(temp_.count(), bottom_diff, temp_.cpu_data(), bottom_diff);
+}
+
+
+#ifdef CPU_ONLY
+STUB_GPU(BatchNormLayer);
+#endif
+
+INSTANTIATE_CLASS(BatchNormLayer);
+REGISTER_LAYER_CLASS(BatchNorm);
+} // namespace caffe
diff --git a/src/caffe/layers/batch_norm_layer.cu b/src/caffe/layers/batch_norm_layer.cu
new file mode 100644
index 0000000..c21713c
--- /dev/null
+++ b/src/caffe/layers/batch_norm_layer.cu
@@ -0,0 +1,171 @@
+#include <algorithm>
+#include <vector>
+
+#include "caffe/layers/batch_norm_layer.hpp"
+#include "caffe/util/math_functions.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void BatchNormLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ const Dtype* bottom_data = bottom[0]->gpu_data();
+ Dtype* top_data = top[0]->mutable_gpu_data();
+ int num = bottom[0]->shape(0);
+ int spatial_dim = bottom[0]->count()/(channels_*bottom[0]->shape(0));
+
+ if (bottom[0] != top[0]) {
+ caffe_copy(bottom[0]->count(), bottom_data, top_data);
+ }
+
+
+ if (use_global_stats_) {
+ // use the stored mean/variance estimates.
+ const Dtype scale_factor = this->blobs_[2]->cpu_data()[0] == 0 ?
+ 0 : 1 / this->blobs_[2]->cpu_data()[0];
+ caffe_gpu_scale(variance_.count(), scale_factor,
+ this->blobs_[0]->gpu_data(), mean_.mutable_gpu_data());
+ caffe_gpu_scale(variance_.count(), scale_factor,
+ this->blobs_[1]->gpu_data(), variance_.mutable_gpu_data());
+ } else {
+ // compute mean
+ caffe_gpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim,
+ 1. / (num * spatial_dim), bottom_data,
+ spatial_sum_multiplier_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.gpu_data(), batch_sum_multiplier_.gpu_data(), 0.,
+ mean_.mutable_gpu_data());
+ }
+
+ // subtract mean
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.gpu_data(), mean_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
+ spatial_dim, 1, -1, num_by_chans_.gpu_data(),
+ spatial_sum_multiplier_.gpu_data(), 1., top_data);
+
+ if (!use_global_stats_) {
+ // compute variance using var(X) = E((X-EX)^2)
+ caffe_gpu_powx(top[0]->count(), top_data, Dtype(2),
+ temp_.mutable_gpu_data()); // (X-EX)^2
+ caffe_gpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim,
+ 1. / (num * spatial_dim), temp_.gpu_data(),
+ spatial_sum_multiplier_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.gpu_data(), batch_sum_multiplier_.gpu_data(), 0.,
+ variance_.mutable_gpu_data()); // E((X_EX)^2)
+
+ // compute and save moving average
+ this->blobs_[2]->mutable_cpu_data()[0] *= moving_average_fraction_;
+ this->blobs_[2]->mutable_cpu_data()[0] += 1;
+ caffe_gpu_axpby(mean_.count(), Dtype(1), mean_.gpu_data(),
+ moving_average_fraction_, this->blobs_[0]->mutable_gpu_data());
+ int m = bottom[0]->count()/channels_;
+ Dtype bias_correction_factor = m > 1 ? Dtype(m)/(m-1) : 1;
+ caffe_gpu_axpby(variance_.count(), bias_correction_factor,
+ variance_.gpu_data(), moving_average_fraction_,
+ this->blobs_[1]->mutable_gpu_data());
+ }
+
+ // normalize variance
+ caffe_gpu_add_scalar(variance_.count(), eps_, variance_.mutable_gpu_data());
+ caffe_gpu_powx(variance_.count(), variance_.gpu_data(), Dtype(0.5),
+ variance_.mutable_gpu_data());
+
+ // replicate variance to input size
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.gpu_data(), variance_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
+ spatial_dim, 1, 1., num_by_chans_.gpu_data(),
+ spatial_sum_multiplier_.gpu_data(), 0., temp_.mutable_gpu_data());
+ caffe_gpu_div(temp_.count(), top_data, temp_.gpu_data(), top_data);
+ // TODO(cdoersch): The caching is only needed because later in-place layers
+ // might clobber the data. Can we skip this if they won't?
+ caffe_copy(x_norm_.count(), top_data,
+ x_norm_.mutable_gpu_data());
+}
+
+template <typename Dtype>
+void BatchNormLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
+ const vector<bool>& propagate_down,
+ const vector<Blob<Dtype>*>& bottom) {
+ const Dtype* top_diff;
+ if (bottom[0] != top[0]) {
+ top_diff = top[0]->gpu_diff();
+ } else {
+ caffe_copy(x_norm_.count(), top[0]->gpu_diff(), x_norm_.mutable_gpu_diff());
+ top_diff = x_norm_.gpu_diff();
+ }
+ Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
+ if (use_global_stats_) {
+ caffe_gpu_div(temp_.count(), top_diff, temp_.gpu_data(), bottom_diff);
+ return;
+ }
+ const Dtype* top_data = x_norm_.gpu_data();
+ int num = bottom[0]->shape()[0];
+ int spatial_dim = bottom[0]->count()/(channels_*bottom[0]->shape(0));
+ // if Y = (X-mean(X))/(sqrt(var(X)+eps)), then
+ //
+ // dE(Y)/dX =
+ // (dE/dY - mean(dE/dY) - mean(dE/dY \cdot Y) \cdot Y)
+ // ./ sqrt(var(X) + eps)
+ //
+ // where \cdot and ./ are hadamard product and elementwise division,
+ // respectively, dE/dY is the top diff, and mean/var/sum are all computed
+ // along all dimensions except the channels dimension. In the above
+ // equation, the operations allow for expansion (i.e. broadcast) along all
+ // dimensions except the channels dimension where required.
+
+ // sum(dE/dY \cdot Y)
+ caffe_gpu_mul(temp_.count(), top_data, top_diff, bottom_diff);
+ caffe_gpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim, 1.,
+ bottom_diff, spatial_sum_multiplier_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.gpu_data(), batch_sum_multiplier_.gpu_data(), 0.,
+ mean_.mutable_gpu_data());
+
+ // reshape (broadcast) the above
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.gpu_data(), mean_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
+ spatial_dim, 1, 1., num_by_chans_.gpu_data(),
+ spatial_sum_multiplier_.gpu_data(), 0., bottom_diff);
+
+ // sum(dE/dY \cdot Y) \cdot Y
+ caffe_gpu_mul(temp_.count(), top_data, bottom_diff, bottom_diff);
+
+ // sum(dE/dY)-sum(dE/dY \cdot Y) \cdot Y
+ caffe_gpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim, 1.,
+ top_diff, spatial_sum_multiplier_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
+ num_by_chans_.gpu_data(), batch_sum_multiplier_.gpu_data(), 0.,
+ mean_.mutable_gpu_data());
+ // reshape (broadcast) the above to make
+ // sum(dE/dY)-sum(dE/dY \cdot Y) \cdot Y
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
+ batch_sum_multiplier_.gpu_data(), mean_.gpu_data(), 0.,
+ num_by_chans_.mutable_gpu_data());
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num * channels_,
+ spatial_dim, 1, 1., num_by_chans_.gpu_data(),
+ spatial_sum_multiplier_.gpu_data(), 1., bottom_diff);
+
+ // dE/dY - mean(dE/dY)-mean(dE/dY \cdot Y) \cdot Y
+ caffe_gpu_axpby(temp_.count(), Dtype(1), top_diff,
+ Dtype(-1. / (num * spatial_dim)), bottom_diff);
+
+ // note: temp_ still contains sqrt(var(X)+eps), computed during the forward
+ // pass.
+ caffe_gpu_div(temp_.count(), bottom_diff, temp_.gpu_data(), bottom_diff);
+}
+
+INSTANTIATE_LAYER_GPU_FUNCS(BatchNormLayer);
+
+
+} // namespace caffe
diff --git a/src/caffe/layers/batch_reindex_layer.cpp b/src/caffe/layers/batch_reindex_layer.cpp
new file mode 100644
index 0000000..b14e56f
--- /dev/null
+++ b/src/caffe/layers/batch_reindex_layer.cpp
@@ -0,0 +1,78 @@
+#include <vector>
+
+#include "caffe/layers/batch_reindex_layer.hpp"
+#include "caffe/util/math_functions.hpp"
+
+namespace caffe {
+
+template<typename Dtype>
+void BatchReindexLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ CHECK_EQ(1, bottom[1]->num_axes());
+ vector<int> newshape;
+ newshape.push_back(bottom[1]->shape(0));
+ for (int i = 1; i < bottom[0]->shape().size(); ++i) {
+ newshape.push_back(bottom[0]->shape()[i]);
+ }
+ top[0]->Reshape(newshape);
+}
+
+template<typename Dtype>
+void BatchReindexLayer<Dtype>::check_batch_reindex(int initial_num,
+ int final_num,
+ const Dtype* ridx_data) {
+ for (int i = 0; i < final_num; ++i) {
+ CHECK_GE(ridx_data[i], 0)
+ << "Index specified for reindex layer was negative.";
+ CHECK_LT(ridx_data[i], initial_num)
+ << "Index specified for reindex layer was greater than batch size.";
+ }
+}
+
+template<typename Dtype>
+void BatchReindexLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ check_batch_reindex(bottom[0]->shape(0), bottom[1]->count(),
+ bottom[1]->cpu_data());
+ if (top[0]->count() == 0) {
+ return;
+ }
+ int inner_dim = bottom[0]->count() / bottom[0]->shape(0);
+ const Dtype* in = bottom[0]->cpu_data();
+ const Dtype* permut = bottom[1]->cpu_data();
+ Dtype* out = top[0]->mutable_cpu_data();
+ for (int index = 0; index < top[0]->count(); ++index) {
+ int n = index / (inner_dim);
+ int in_n = static_cast<int>(permut[n]);
+ out[index] = in[in_n * (inner_dim) + index % (inner_dim)];
+ }
+}
+
+template<typename Dtype>
+void BatchReindexLayer<Dtype>::Backward_cpu(
+ const vector<Blob<Dtype>*>& top, const vector<bool>& propagate_down,
+ const vector<Blob<Dtype>*>& bottom) {
+ CHECK(!propagate_down[1]) << "Cannot backprop to index.";
+ if (!propagate_down[0]) {
+ return;
+ }
+ int inner_dim = bottom[0]->count() / bottom[0]->shape(0);
+ Dtype* bot_diff = bottom[0]->mutable_cpu_diff();
+ const Dtype* permut = bottom[1]->cpu_data();
+ const Dtype* top_diff = top[0]->cpu_diff();
+ caffe_set(bottom[0]->count(), Dtype(0), bot_diff);
+ for (int index = 0; index < top[0]->count(); ++index) {
+ int n = index / (inner_dim);
+ int in_n = static_cast<int>(permut[n]);
+ bot_diff[in_n * (inner_dim) + index % (inner_dim)] += top_diff[index];
+ }
+}
+
+#ifdef CPU_ONLY
+STUB_GPU(BatchReindexLayer);
+#endif
+
+INSTANTIATE_CLASS(BatchReindexLayer);
+REGISTER_LAYER_CLASS(BatchReindex);
+
+} // namespace caffe
diff --git a/src/caffe/layers/batch_reindex_layer.cu b/src/caffe/layers/batch_reindex_layer.cu
new file mode 100644
index 0000000..83054d3
--- /dev/null
+++ b/src/caffe/layers/batch_reindex_layer.cu
@@ -0,0 +1,106 @@
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+#include "caffe/layers/batch_reindex_layer.hpp"
+#include "caffe/util/math_functions.hpp"
+
+namespace caffe {
+
+template<typename Dtype>
+__global__ void BRForward(const int count, const int inner_dim, const Dtype* in,
+ const Dtype* permut, Dtype* out) {
+ CUDA_KERNEL_LOOP(index, count) {
+ int n = index / (inner_dim);
+ int in_n = static_cast<int>(permut[n]);
+ out[index] = in[in_n * (inner_dim) + index % (inner_dim)];
+ }
+}
+
+template<typename Dtype>
+void BatchReindexLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ check_batch_reindex(bottom[0]->shape(0), bottom[1]->count(),
+ bottom[1]->cpu_data());
+ if (top[0]->count() == 0) {
+ return;
+ }
+ int threads = top[0]->count();
+ // NOLINT_NEXT_LINE(whitespace/operators)
+ BRForward<Dtype> <<<CAFFE_GET_BLOCKS(threads), CAFFE_CUDA_NUM_THREADS>>>(
+ top[0]->count(), bottom[0]->count() / bottom[0]->shape(0),
+ bottom[0]->gpu_data(), bottom[1]->gpu_data(), top[0]->mutable_gpu_data());
+ CUDA_POST_KERNEL_CHECK;
+}
+
+template<typename Dtype>
+__global__ void BRBackward(const int count, const int inner_dim,
+ const Dtype* in, const Dtype* top_indexes,
+ const Dtype* begins, const Dtype* counts,
+ Dtype* out) {
+ CUDA_KERNEL_LOOP(index, count) {
+ int n = index / (inner_dim);
+ out[index] = 0;
+ int lower = static_cast<int>(begins[n]);
+ int upper = lower + static_cast<int>(counts[n]);
+ for (int i = lower; i < upper; ++i) {
+ int in_n = static_cast<int>(top_indexes[i]);
+ out[index] += in[in_n * (inner_dim) + index % (inner_dim)];
+ }
+ }
+}
+
+template<typename Dtype>
+void BatchReindexLayer<Dtype>::Backward_gpu(
+ const vector<Blob<Dtype>*>& top, const vector<bool>& propagate_down,
+ const vector<Blob<Dtype>*>& bottom) {
+ CHECK(!propagate_down[1]) << "Cannot backprop to index.";
+ if (!propagate_down[0]) {
+ return;
+ }
+
+ vector<std::pair<int, int> > mapping;
+ const Dtype* perm = bottom[1]->cpu_data();
+ for (int i = 0; i < bottom[1]->count(); ++i) {
+ mapping.push_back(pair<int, int>(static_cast<int>(perm[i]), i));
+ }
+ std::sort(mapping.begin(), mapping.end(), pair_sort_first());
+
+ // Each element of the bottom diff is potentially the sum of many top diffs.
+ // However, we'd like each CUDA thread to handle exactly one output. Hence,
+ // we first pre-compute a list of lists of indices that need to be summed for
+ // each output. `top_indexes` holds the data of this list of lists. The
+ // k'th element of `begins` points to the location in `top_indexes` where the
+ // list for the k'th example begin, and the k'th element of `counts` is the
+ // length of that list.
+ vector<int> shape;
+ shape.push_back(bottom[1]->count());
+ Blob<Dtype> top_indexes(shape);
+ shape[0] = bottom[0]->shape(0);
+ Blob<Dtype> counts(shape);
+ Blob<Dtype> begins(shape);
+ Dtype* t_i_data = top_indexes.mutable_cpu_data();
+ Dtype* c_data = counts.mutable_cpu_data();
+ Dtype* b_data = begins.mutable_cpu_data();
+ caffe_set(begins.count(), Dtype(-1), b_data);
+ caffe_set(counts.count(), Dtype(0), c_data);
+ for (int i = 0; i < mapping.size(); ++i) {
+ t_i_data[i] = mapping[i].second;
+ if (b_data[mapping[i].first] == -1) {
+ b_data[mapping[i].first] = i;
+ }
+ c_data[mapping[i].first] += 1;
+ }
+
+ int threads = bottom[0]->count();
+ // NOLINT_NEXT_LINE(whitespace/operators)
+ BRBackward<Dtype> <<<CAFFE_GET_BLOCKS(threads), CAFFE_CUDA_NUM_THREADS>>>(
+ bottom[0]->count(), bottom[0]->count() / bottom[0]->shape(0),
+ top[0]->gpu_diff(), top_indexes.gpu_data(), begins.gpu_data(),
+ counts.gpu_data(), bottom[0]->mutable_gpu_diff());
+ CUDA_POST_KERNEL_CHECK;
+}
+
+INSTANTIATE_LAYER_GPU_FUNCS(BatchReindexLayer);
+
+} // namespace caffe
diff --git a/src/caffe/layers/bnll_layer.cpp b/src/caffe/layers/bnll_layer.cpp
index 9ba0ea9..448d86d 100644
--- a/src/caffe/layers/bnll_layer.cpp
+++ b/src/caffe/layers/bnll_layer.cpp
@@ -1,8 +1,7 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/bnll_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/bnll_layer.cu b/src/caffe/layers/bnll_layer.cu
index d963d06..8df8ef0 100644
--- a/src/caffe/layers/bnll_layer.cu
+++ b/src/caffe/layers/bnll_layer.cu
@@ -1,8 +1,7 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/bnll_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/concat_layer.cpp b/src/caffe/layers/concat_layer.cpp
index 86b500d..580bd47 100644
--- a/src/caffe/layers/concat_layer.cpp
+++ b/src/caffe/layers/concat_layer.cpp
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/concat_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/concat_layer.cu b/src/caffe/layers/concat_layer.cu
index 617701e..a3a0bf6 100644
--- a/src/caffe/layers/concat_layer.cu
+++ b/src/caffe/layers/concat_layer.cu
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/concat_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/contrastive_loss_layer.cpp b/src/caffe/layers/contrastive_loss_layer.cpp
index 25e1678..599e178 100644
--- a/src/caffe/layers/contrastive_loss_layer.cpp
+++ b/src/caffe/layers/contrastive_loss_layer.cpp
@@ -1,9 +1,7 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/loss_layers.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/contrastive_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
@@ -53,7 +51,8 @@ void ContrastiveLossLayer<Dtype>::Forward_cpu(
if (legacy_version) {
loss += std::max(margin - dist_sq_.cpu_data()[i], Dtype(0.0));
} else {
- Dtype dist = std::max(margin - sqrt(dist_sq_.cpu_data()[i]), 0.0);
+ Dtype dist = std::max<Dtype>(margin - sqrt(dist_sq_.cpu_data()[i]),
+ Dtype(0.0));
loss += dist*dist;
}
}
diff --git a/src/caffe/layers/contrastive_loss_layer.cu b/src/caffe/layers/contrastive_loss_layer.cu
index 9312393..fd7d67c 100644
--- a/src/caffe/layers/contrastive_loss_layer.cu
+++ b/src/caffe/layers/contrastive_loss_layer.cu
@@ -1,10 +1,8 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/contrastive_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/conv_layer.cpp b/src/caffe/layers/conv_layer.cpp
index fb50bb0..cff0978 100644
--- a/src/caffe/layers/conv_layer.cpp
+++ b/src/caffe/layers/conv_layer.cpp
@@ -1,10 +1,6 @@
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/conv_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/conv_layer.cu b/src/caffe/layers/conv_layer.cu
index b429d2b..d06e4b6 100644
--- a/src/caffe/layers/conv_layer.cu
+++ b/src/caffe/layers/conv_layer.cu
@@ -1,10 +1,6 @@
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/conv_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_conv_layer.cpp b/src/caffe/layers/cudnn_conv_layer.cpp
index 3514fe2..1987fb0 100644
--- a/src/caffe/layers/cudnn_conv_layer.cpp
+++ b/src/caffe/layers/cudnn_conv_layer.cpp
@@ -1,11 +1,8 @@
#ifdef USE_CUDNN
+#include <algorithm>
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_conv_layer.hpp"
namespace caffe {
@@ -24,13 +21,38 @@ void CuDNNConvolutionLayer<Dtype>::LayerSetUp(
// Initialize CUDA streams and cuDNN.
stream_ = new cudaStream_t[this->group_ * CUDNN_STREAMS_PER_GROUP];
handle_ = new cudnnHandle_t[this->group_ * CUDNN_STREAMS_PER_GROUP];
+
+ // Initialize algorithm arrays
+ fwd_algo_ = new cudnnConvolutionFwdAlgo_t[bottom.size()];
+ bwd_filter_algo_= new cudnnConvolutionBwdFilterAlgo_t[bottom.size()];
+ bwd_data_algo_ = new cudnnConvolutionBwdDataAlgo_t[bottom.size()];
+
+ // initialize size arrays
+ workspace_fwd_sizes_ = new size_t[bottom.size()];
+ workspace_bwd_filter_sizes_ = new size_t[bottom.size()];
+ workspace_bwd_data_sizes_ = new size_t[bottom.size()];
+
+ // workspace data
workspaceSizeInBytes = 0;
- workspace = NULL;
+ workspaceData = NULL;
+ workspace = new void*[this->group_ * CUDNN_STREAMS_PER_GROUP];
+
+ for (size_t i = 0; i < bottom.size(); ++i) {
+ // initialize all to default algorithms
+ fwd_algo_[i] = (cudnnConvolutionFwdAlgo_t)0;
+ bwd_filter_algo_[i] = (cudnnConvolutionBwdFilterAlgo_t)0;
+ bwd_data_algo_[i] = (cudnnConvolutionBwdDataAlgo_t)0;
+ // default algorithms don't require workspace
+ workspace_fwd_sizes_[i] = 0;
+ workspace_bwd_data_sizes_[i] = 0;
+ workspace_bwd_filter_sizes_[i] = 0;
+ }
for (int g = 0; g < this->group_ * CUDNN_STREAMS_PER_GROUP; g++) {
CUDA_CHECK(cudaStreamCreate(&stream_[g]));
CUDNN_CHECK(cudnnCreate(&handle_[g]));
CUDNN_CHECK(cudnnSetStream(handle_[g], stream_[g]));
+ workspace[g] = NULL;
}
// Set the indexing parameters.
@@ -86,6 +108,10 @@ void CuDNNConvolutionLayer<Dtype>::Reshape(
const int stride_h = stride_data[0];
const int stride_w = stride_data[1];
+ // Specify workspace limit for kernels directly until we have a
+ // planning strategy and a rewrite of Caffe's GPU memory mangagement
+ size_t workspace_limit_bytes = 8*1024*1024;
+
for (int i = 0; i < bottom.size(); i++) {
cudnn::setTensor4dDesc<Dtype>(&bottom_descs_[i],
this->num_,
@@ -98,7 +124,104 @@ void CuDNNConvolutionLayer<Dtype>::Reshape(
this->num_output_ * this->out_spatial_dim_,
this->out_spatial_dim_, width_out, 1);
cudnn::setConvolutionDesc<Dtype>(&conv_descs_[i], bottom_descs_[i],
- filter_desc_, pad_h, pad_w, stride_h, stride_w);
+ filter_desc_, pad_h, pad_w,
+ stride_h, stride_w);
+
+ // choose forward and backward algorithms + workspace(s)
+ CUDNN_CHECK(cudnnGetConvolutionForwardAlgorithm(handle_[0],
+ bottom_descs_[i],
+ filter_desc_,
+ conv_descs_[i],
+ top_descs_[i],
+ CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT,
+ workspace_limit_bytes,
+ &fwd_algo_[i]));
+
+ CUDNN_CHECK(cudnnGetConvolutionForwardWorkspaceSize(handle_[0],
+ bottom_descs_[i],
+ filter_desc_,
+ conv_descs_[i],
+ top_descs_[i],
+ fwd_algo_[i],
+ &(workspace_fwd_sizes_[i])));
+
+ // choose backward algorithm for filter
+ CUDNN_CHECK(cudnnGetConvolutionBackwardFilterAlgorithm(handle_[0],
+ bottom_descs_[i], top_descs_[i], conv_descs_[i], filter_desc_,
+ CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT,
+ workspace_limit_bytes, &bwd_filter_algo_[i]) );
+
+ // get workspace for backwards filter algorithm
+ CUDNN_CHECK(cudnnGetConvolutionBackwardFilterWorkspaceSize(handle_[0],
+ bottom_descs_[i], top_descs_[i], conv_descs_[i], filter_desc_,
+ bwd_filter_algo_[i], &workspace_bwd_filter_sizes_[i]));
+
+ // choose backward algo for data
+ CUDNN_CHECK(cudnnGetConvolutionBackwardDataAlgorithm(handle_[0],
+ filter_desc_, top_descs_[i], conv_descs_[i], bottom_descs_[i],
+ CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT,
+ workspace_limit_bytes, &bwd_data_algo_[i]));
+
+ // get workspace size
+ CUDNN_CHECK(cudnnGetConvolutionBackwardDataWorkspaceSize(handle_[0],
+ filter_desc_, top_descs_[i], conv_descs_[i], bottom_descs_[i],
+ bwd_data_algo_[i], &workspace_bwd_data_sizes_[i]) );
+ }
+
+ // reduce over all workspace sizes to get a maximum to allocate / reallocate
+ size_t total_workspace_fwd = 0;
+ size_t total_workspace_bwd_data = 0;
+ size_t total_workspace_bwd_filter = 0;
+
+ for (size_t i = 0; i < bottom.size(); i++) {
+ total_workspace_fwd = std::max(total_workspace_fwd,
+ workspace_fwd_sizes_[i]);
+ total_workspace_bwd_data = std::max(total_workspace_bwd_data,
+ workspace_bwd_data_sizes_[i]);
+ total_workspace_bwd_filter = std::max(total_workspace_bwd_filter,
+ workspace_bwd_filter_sizes_[i]);
+ }
+ // get max over all operations
+ size_t max_workspace = std::max(total_workspace_fwd,
+ total_workspace_bwd_data);
+ max_workspace = std::max(max_workspace, total_workspace_bwd_filter);
+ // ensure all groups have enough workspace
+ size_t total_max_workspace = max_workspace *
+ (this->group_ * CUDNN_STREAMS_PER_GROUP);
+
+ // this is the total amount of storage needed over all groups + streams
+ if (total_max_workspace > workspaceSizeInBytes) {
+ DLOG(INFO) << "Reallocating workspace storage: " << total_max_workspace;
+ workspaceSizeInBytes = total_max_workspace;
+
+ // free the existing workspace and allocate a new (larger) one
+ cudaFree(this->workspaceData);
+
+ cudaError_t err = cudaMalloc(&(this->workspaceData), workspaceSizeInBytes);
+ if (err != cudaSuccess) {
+ // force zero memory path
+ for (int i = 0; i < bottom.size(); i++) {
+ workspace_fwd_sizes_[i] = 0;
+ workspace_bwd_filter_sizes_[i] = 0;
+ workspace_bwd_data_sizes_[i] = 0;
+ fwd_algo_[i] = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM;
+ bwd_filter_algo_[i] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0;
+ bwd_data_algo_[i] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_0;
+ }
+
+ // NULL out all workspace pointers
+ for (int g = 0; g < (this->group_ * CUDNN_STREAMS_PER_GROUP); g++) {
+ workspace[g] = NULL;
+ }
+ // NULL out underlying data
+ workspaceData = NULL;
+ workspaceSizeInBytes = 0;
+ }
+
+ // if we succeed in the allocation, set pointer aliases for workspaces
+ for (int g = 0; g < (this->group_ * CUDNN_STREAMS_PER_GROUP); g++) {
+ workspace[g] = reinterpret_cast<char *>(workspaceData) + g*max_workspace;
+ }
}
// Tensor descriptor for bias.
@@ -128,8 +251,15 @@ CuDNNConvolutionLayer<Dtype>::~CuDNNConvolutionLayer() {
cudnnDestroy(handle_[g]);
}
+ cudaFree(workspaceData);
delete [] stream_;
delete [] handle_;
+ delete [] fwd_algo_;
+ delete [] bwd_filter_algo_;
+ delete [] bwd_data_algo_;
+ delete [] workspace_fwd_sizes_;
+ delete [] workspace_bwd_data_sizes_;
+ delete [] workspace_bwd_filter_sizes_;
}
INSTANTIATE_CLASS(CuDNNConvolutionLayer);
diff --git a/src/caffe/layers/cudnn_conv_layer.cu b/src/caffe/layers/cudnn_conv_layer.cu
index 6911520..42c4fd0 100644
--- a/src/caffe/layers/cudnn_conv_layer.cu
+++ b/src/caffe/layers/cudnn_conv_layer.cu
@@ -1,11 +1,7 @@
#ifdef USE_CUDNN
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_conv_layer.hpp"
namespace caffe {
@@ -14,11 +10,6 @@ __global__ void sync_conv_groups() { }
template <typename Dtype>
void CuDNNConvolutionLayer<Dtype>::Forward_gpu(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
- const int* kernel_shape_data = this->kernel_shape_.cpu_data();
- const int kernel_h = kernel_shape_data[0];
- const int kernel_w = kernel_shape_data[1];
- const size_t workspace_limit_bytes =
- kernel_h * kernel_w * this->channels_ * sizeof(int) + 1;
const Dtype* weight = this->blobs_[0]->gpu_data();
for (int i = 0; i < bottom.size(); ++i) {
const Dtype* bottom_data = bottom[i]->gpu_data();
@@ -26,63 +17,32 @@ void CuDNNConvolutionLayer<Dtype>::Forward_gpu(
// Forward through cuDNN in parallel over groups.
for (int g = 0; g < this->group_; g++) {
- cudnnConvolutionFwdAlgo_t algo;
-
- // pick the convolution algorithm
- // TODO(shelhamer) this should be done during reshape
- // TODO(shelhamer) the choice of automatic or manual algorithm picking
- // should be exposed in proto
- CUDNN_CHECK(cudnnGetConvolutionForwardAlgorithm(handle_[g],
- bottom_descs_[i],
- filter_desc_,
- conv_descs_[i],
- top_descs_[i],
- CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT,
- workspace_limit_bytes, // memoryLimitInBytes,
- &algo));
-
- // get minimum size of the workspace needed for the desired algorithm
- size_t workspaceSizeInBytes_temp = 0;
-
- CUDNN_CHECK(cudnnGetConvolutionForwardWorkspaceSize(handle_[g],
- bottom_descs_[i],
- filter_desc_,
- conv_descs_[i],
- top_descs_[i],
- algo,
- &workspaceSizeInBytes_temp));
-
- if (workspaceSizeInBytes_temp > workspaceSizeInBytes) {
- workspaceSizeInBytes = workspaceSizeInBytes_temp;
- // free the existing workspace and allocate a new (larger) one
- cudaFree(this->workspace);
- cudaError_t err = cudaMalloc(&(this->workspace), workspaceSizeInBytes);
- if (err != cudaSuccess) {
- // force zero memory path
- algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM;
- workspace = NULL;
- workspaceSizeInBytes = 0;
- }
- }
-
// Filters.
CUDNN_CHECK(cudnnConvolutionForward(handle_[g],
cudnn::dataType<Dtype>::one,
bottom_descs_[i], bottom_data + bottom_offset_ * g,
filter_desc_, weight + this->weight_offset_ * g,
conv_descs_[i],
- algo, workspace, workspaceSizeInBytes,
+ fwd_algo_[i], workspace[g], workspace_fwd_sizes_[i],
cudnn::dataType<Dtype>::zero,
top_descs_[i], top_data + top_offset_ * g));
// Bias.
if (this->bias_term_) {
const Dtype* bias_data = this->blobs_[1]->gpu_data();
+#if CUDNN_VERSION_MIN(4, 0, 0)
+ CUDNN_CHECK(cudnnAddTensor(handle_[g],
+ cudnn::dataType<Dtype>::one,
+ bias_desc_, bias_data + bias_offset_ * g,
+ cudnn::dataType<Dtype>::one,
+ top_descs_[i], top_data + top_offset_ * g));
+#else
CUDNN_CHECK(cudnnAddTensor(handle_[g], CUDNN_ADD_SAME_C,
cudnn::dataType<Dtype>::one,
bias_desc_, bias_data + bias_offset_ * g,
cudnn::dataType<Dtype>::one,
top_descs_[i], top_data + top_offset_ * g));
+#endif
}
}
@@ -122,11 +82,14 @@ void CuDNNConvolutionLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
// Gradient w.r.t. weights.
if (this->param_propagate_down_[0]) {
const Dtype* bottom_data = bottom[i]->gpu_data();
- CUDNN_CHECK(cudnnConvolutionBackwardFilter(handle_[1*this->group_ + g],
+ CUDNN_CHECK(cudnnConvolutionBackwardFilter_v3(
+ handle_[1*this->group_ + g],
cudnn::dataType<Dtype>::one,
bottom_descs_[i], bottom_data + bottom_offset_ * g,
top_descs_[i], top_diff + top_offset_ * g,
conv_descs_[i],
+ bwd_filter_algo_[i], workspace[1*this->group_ + g],
+ workspace_bwd_filter_sizes_[i],
cudnn::dataType<Dtype>::one,
filter_desc_, weight_diff + this->weight_offset_ * g));
}
@@ -137,11 +100,14 @@ void CuDNNConvolutionLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
weight = this->blobs_[0]->gpu_data();
}
Dtype* bottom_diff = bottom[i]->mutable_gpu_diff();
- CUDNN_CHECK(cudnnConvolutionBackwardData(handle_[2*this->group_ + g],
+ CUDNN_CHECK(cudnnConvolutionBackwardData_v3(
+ handle_[2*this->group_ + g],
cudnn::dataType<Dtype>::one,
filter_desc_, weight + this->weight_offset_ * g,
top_descs_[i], top_diff + top_offset_ * g,
conv_descs_[i],
+ bwd_data_algo_[i], workspace[2*this->group_ + g],
+ workspace_bwd_data_sizes_[i],
cudnn::dataType<Dtype>::zero,
bottom_descs_[i], bottom_diff + bottom_offset_ * g));
}
diff --git a/src/caffe/layers/cudnn_lcn_layer.cpp b/src/caffe/layers/cudnn_lcn_layer.cpp
new file mode 100644
index 0000000..9c09bf2
--- /dev/null
+++ b/src/caffe/layers/cudnn_lcn_layer.cpp
@@ -0,0 +1,73 @@
+#ifdef USE_CUDNN
+#include <vector>
+
+#include "caffe/layers/cudnn_lcn_layer.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void CuDNNLCNLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ LRNLayer<Dtype>::LayerSetUp(bottom, top);
+
+ CUDNN_CHECK(cudnnCreate(&handle_));
+ CUDNN_CHECK(cudnnCreateLRNDescriptor(&norm_desc_));
+ cudnn::createTensor4dDesc<Dtype>(&bottom_desc_);
+ cudnn::createTensor4dDesc<Dtype>(&top_desc_);
+
+ // create a LRN handle
+ handles_setup_ = true;
+
+ size_ = this->layer_param().lrn_param().local_size();
+ pre_pad_ = (size_ - 1) / 2;
+ alpha_ = this->layer_param().lrn_param().alpha();
+ beta_ = this->layer_param().lrn_param().beta();
+ k_ = this->layer_param().lrn_param().k();
+}
+
+template <typename Dtype>
+void CuDNNLCNLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ LRNLayer<Dtype>::Reshape(bottom, top);
+ cudnn::setTensor4dDesc<Dtype>(&bottom_desc_, bottom[0]->num(),
+ this->channels_, this->height_, this->width_);
+ cudnn::setTensor4dDesc<Dtype>(&top_desc_, bottom[0]->num(),
+ this->channels_, this->height_, this->width_);
+ CUDNN_CHECK(cudnnSetLRNDescriptor(norm_desc_, size_, alpha_, beta_, k_));
+
+ // allocate / reallocate tempData buffers
+ size_t totalSizeInBytes = sizeof(Dtype)*bottom[0]->num()* \
+ this->channels_*this->height_*this->width_;
+
+ if (totalSizeInBytes > tempDataSize) {
+ tempDataSize = totalSizeInBytes;
+
+ cudaFree(tempData1);
+ cudaFree(tempData2);
+
+ // allocate new buffers
+ CUDA_CHECK(cudaMalloc(&tempData1, totalSizeInBytes));
+ CUDA_CHECK(cudaMalloc(&tempData2, totalSizeInBytes));
+ }
+}
+
+template <typename Dtype>
+CuDNNLCNLayer<Dtype>::~CuDNNLCNLayer() {
+ // Check that handles have been setup before destroying.
+ if (!handles_setup_) { return; }
+
+ cudnnDestroyTensorDescriptor(bottom_desc_);
+ cudnnDestroyTensorDescriptor(top_desc_);
+
+ // destroy LRN handle
+ cudnnDestroy(handle_);
+
+ // free temp buffers
+ cudaFree(tempData1);
+ cudaFree(tempData2);
+}
+
+INSTANTIATE_CLASS(CuDNNLCNLayer);
+
+} // namespace caffe
+#endif
diff --git a/src/caffe/layers/cudnn_pooling_layer.cu b/src/caffe/layers/cudnn_lcn_layer.cu
similarity index 52%
copy from src/caffe/layers/cudnn_pooling_layer.cu
copy to src/caffe/layers/cudnn_lcn_layer.cu
index a952b85..b44ef47 100644
--- a/src/caffe/layers/cudnn_pooling_layer.cu
+++ b/src/caffe/layers/cudnn_lcn_layer.cu
@@ -1,45 +1,46 @@
#ifdef USE_CUDNN
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_lcn_layer.hpp"
namespace caffe {
template <typename Dtype>
-void CuDNNPoolingLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+void CuDNNLCNLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const Dtype* bottom_data = bottom[0]->gpu_data();
Dtype* top_data = top[0]->mutable_gpu_data();
- CUDNN_CHECK(cudnnPoolingForward(handle_, pooling_desc_,
+
+ CUDNN_CHECK(cudnnDivisiveNormalizationForward(
+ handle_, norm_desc_, CUDNN_DIVNORM_PRECOMPUTED_MEANS,
cudnn::dataType<Dtype>::one,
bottom_desc_, bottom_data,
+ NULL, // srcMeansData
+ this->tempData1, this->tempData2,
cudnn::dataType<Dtype>::zero,
- top_desc_, top_data));
+ top_desc_, top_data) );
}
template <typename Dtype>
-void CuDNNPoolingLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
+void CuDNNLCNLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
- if (!propagate_down[0]) {
- return;
- }
const Dtype* top_diff = top[0]->gpu_diff();
const Dtype* top_data = top[0]->gpu_data();
const Dtype* bottom_data = bottom[0]->gpu_data();
Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
- CUDNN_CHECK(cudnnPoolingBackward(handle_, pooling_desc_,
+
+ CUDNN_CHECK(cudnnDivisiveNormalizationBackward(
+ handle_, norm_desc_, CUDNN_DIVNORM_PRECOMPUTED_MEANS,
cudnn::dataType<Dtype>::one,
- top_desc_, top_data, top_desc_, top_diff,
bottom_desc_, bottom_data,
+ NULL, top_diff, // NULL - srcMeansData
+ this->tempData1, this->tempData2,
cudnn::dataType<Dtype>::zero,
- bottom_desc_, bottom_diff));
+ bottom_desc_, bottom_diff,
+ NULL) );
}
-INSTANTIATE_LAYER_GPU_FUNCS(CuDNNPoolingLayer);
+INSTANTIATE_LAYER_GPU_FUNCS(CuDNNLCNLayer);
} // namespace caffe
#endif
diff --git a/src/caffe/layers/cudnn_lrn_layer.cpp b/src/caffe/layers/cudnn_lrn_layer.cpp
new file mode 100644
index 0000000..0495b80
--- /dev/null
+++ b/src/caffe/layers/cudnn_lrn_layer.cpp
@@ -0,0 +1,53 @@
+#ifdef USE_CUDNN
+#include <vector>
+
+#include "caffe/layers/cudnn_lrn_layer.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void CuDNNLRNLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ LRNLayer<Dtype>::LayerSetUp(bottom, top);
+
+ CUDNN_CHECK(cudnnCreate(&handle_));
+ CUDNN_CHECK(cudnnCreateLRNDescriptor(&norm_desc_));
+ cudnn::createTensor4dDesc<Dtype>(&bottom_desc_);
+ cudnn::createTensor4dDesc<Dtype>(&top_desc_);
+
+ // create a LRN handle
+ handles_setup_ = true;
+
+ size_ = this->layer_param().lrn_param().local_size();
+ alpha_ = this->layer_param().lrn_param().alpha();
+ beta_ = this->layer_param().lrn_param().beta();
+ k_ = this->layer_param().lrn_param().k();
+}
+
+template <typename Dtype>
+void CuDNNLRNLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
+ const vector<Blob<Dtype>*>& top) {
+ LRNLayer<Dtype>::Reshape(bottom, top);
+ cudnn::setTensor4dDesc<Dtype>(&bottom_desc_, bottom[0]->num(),
+ this->channels_, this->height_, this->width_);
+ cudnn::setTensor4dDesc<Dtype>(&top_desc_, bottom[0]->num(),
+ this->channels_, this->height_, this->width_);
+ CUDNN_CHECK(cudnnSetLRNDescriptor(norm_desc_, size_, alpha_, beta_, k_));
+}
+
+template <typename Dtype>
+CuDNNLRNLayer<Dtype>::~CuDNNLRNLayer() {
+ // Check that handles have been setup before destroying.
+ if (!handles_setup_) { return; }
+
+ cudnnDestroyTensorDescriptor(bottom_desc_);
+ cudnnDestroyTensorDescriptor(top_desc_);
+
+ // destroy LRN handle
+ cudnnDestroy(handle_);
+}
+
+INSTANTIATE_CLASS(CuDNNLRNLayer);
+
+} // namespace caffe
+#endif
diff --git a/src/caffe/layers/cudnn_pooling_layer.cu b/src/caffe/layers/cudnn_lrn_layer.cu
similarity index 53%
copy from src/caffe/layers/cudnn_pooling_layer.cu
copy to src/caffe/layers/cudnn_lrn_layer.cu
index a952b85..ca647f3 100644
--- a/src/caffe/layers/cudnn_pooling_layer.cu
+++ b/src/caffe/layers/cudnn_lrn_layer.cu
@@ -1,45 +1,44 @@
#ifdef USE_CUDNN
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_lrn_layer.hpp"
namespace caffe {
template <typename Dtype>
-void CuDNNPoolingLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+void CuDNNLRNLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const Dtype* bottom_data = bottom[0]->gpu_data();
Dtype* top_data = top[0]->mutable_gpu_data();
- CUDNN_CHECK(cudnnPoolingForward(handle_, pooling_desc_,
+
+ CUDNN_CHECK(cudnnLRNCrossChannelForward(
+ handle_, norm_desc_, CUDNN_LRN_CROSS_CHANNEL_DIM1,
cudnn::dataType<Dtype>::one,
bottom_desc_, bottom_data,
cudnn::dataType<Dtype>::zero,
- top_desc_, top_data));
+ top_desc_, top_data) );
}
template <typename Dtype>
-void CuDNNPoolingLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
+void CuDNNLRNLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
- if (!propagate_down[0]) {
- return;
- }
const Dtype* top_diff = top[0]->gpu_diff();
const Dtype* top_data = top[0]->gpu_data();
const Dtype* bottom_data = bottom[0]->gpu_data();
Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
- CUDNN_CHECK(cudnnPoolingBackward(handle_, pooling_desc_,
+
+ CUDNN_CHECK(cudnnLRNCrossChannelBackward(
+ handle_, norm_desc_, CUDNN_LRN_CROSS_CHANNEL_DIM1,
cudnn::dataType<Dtype>::one,
- top_desc_, top_data, top_desc_, top_diff,
+ top_desc_, top_data,
+ top_desc_, top_diff,
bottom_desc_, bottom_data,
cudnn::dataType<Dtype>::zero,
- bottom_desc_, bottom_diff));
+ bottom_desc_, bottom_diff) );
}
-INSTANTIATE_LAYER_GPU_FUNCS(CuDNNPoolingLayer);
+INSTANTIATE_LAYER_GPU_FUNCS(CuDNNLRNLayer);
+
+}; // namespace caffe
-} // namespace caffe
#endif
diff --git a/src/caffe/layers/cudnn_pooling_layer.cpp b/src/caffe/layers/cudnn_pooling_layer.cpp
index c92c4e4..24f1478 100644
--- a/src/caffe/layers/cudnn_pooling_layer.cpp
+++ b/src/caffe/layers/cudnn_pooling_layer.cpp
@@ -1,11 +1,7 @@
#ifdef USE_CUDNN
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_pooling_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_pooling_layer.cu b/src/caffe/layers/cudnn_pooling_layer.cu
index a952b85..6f00195 100644
--- a/src/caffe/layers/cudnn_pooling_layer.cu
+++ b/src/caffe/layers/cudnn_pooling_layer.cu
@@ -1,11 +1,7 @@
#ifdef USE_CUDNN
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_pooling_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_relu_layer.cpp b/src/caffe/layers/cudnn_relu_layer.cpp
index 759d839..c86c690 100644
--- a/src/caffe/layers/cudnn_relu_layer.cpp
+++ b/src/caffe/layers/cudnn_relu_layer.cpp
@@ -1,9 +1,7 @@
#ifdef USE_CUDNN
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_relu_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_relu_layer.cu b/src/caffe/layers/cudnn_relu_layer.cu
index 21d1485..9f61718 100644
--- a/src/caffe/layers/cudnn_relu_layer.cu
+++ b/src/caffe/layers/cudnn_relu_layer.cu
@@ -1,9 +1,7 @@
#ifdef USE_CUDNN
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_relu_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_sigmoid_layer.cpp b/src/caffe/layers/cudnn_sigmoid_layer.cpp
index 3263787..ccb955c 100644
--- a/src/caffe/layers/cudnn_sigmoid_layer.cpp
+++ b/src/caffe/layers/cudnn_sigmoid_layer.cpp
@@ -1,9 +1,7 @@
#ifdef USE_CUDNN
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_sigmoid_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_sigmoid_layer.cu b/src/caffe/layers/cudnn_sigmoid_layer.cu
index 7a06cf7..e2a4b46 100644
--- a/src/caffe/layers/cudnn_sigmoid_layer.cu
+++ b/src/caffe/layers/cudnn_sigmoid_layer.cu
@@ -1,9 +1,7 @@
#ifdef USE_CUDNN
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_sigmoid_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_softmax_layer.cpp b/src/caffe/layers/cudnn_softmax_layer.cpp
index 77a3225..6440df9 100644
--- a/src/caffe/layers/cudnn_softmax_layer.cpp
+++ b/src/caffe/layers/cudnn_softmax_layer.cpp
@@ -1,13 +1,9 @@
#ifdef USE_CUDNN
-#include <algorithm>
-#include <cfloat>
#include <vector>
#include "thrust/device_vector.h"
-#include "caffe/layer.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_softmax_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_softmax_layer.cu b/src/caffe/layers/cudnn_softmax_layer.cu
index a9e2fce..7283eb7 100644
--- a/src/caffe/layers/cudnn_softmax_layer.cu
+++ b/src/caffe/layers/cudnn_softmax_layer.cu
@@ -1,13 +1,9 @@
#ifdef USE_CUDNN
-#include <algorithm>
-#include <cfloat>
#include <vector>
#include "thrust/device_vector.h"
-#include "caffe/layer.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_softmax_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_tanh_layer.cpp b/src/caffe/layers/cudnn_tanh_layer.cpp
index 376faad..1a56418 100644
--- a/src/caffe/layers/cudnn_tanh_layer.cpp
+++ b/src/caffe/layers/cudnn_tanh_layer.cpp
@@ -1,9 +1,7 @@
#ifdef USE_CUDNN
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_tanh_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/cudnn_tanh_layer.cu b/src/caffe/layers/cudnn_tanh_layer.cu
index d287f6f..89df28a 100644
--- a/src/caffe/layers/cudnn_tanh_layer.cu
+++ b/src/caffe/layers/cudnn_tanh_layer.cu
@@ -1,9 +1,7 @@
#ifdef USE_CUDNN
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/cudnn_tanh_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp
index 71f8cb0..66e6301 100644
--- a/src/caffe/layers/data_layer.cpp
+++ b/src/caffe/layers/data_layer.cpp
@@ -3,15 +3,11 @@
#endif // USE_OPENCV
#include <stdint.h>
-#include <string>
#include <vector>
-#include "caffe/common.hpp"
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/proto/caffe.pb.h"
+#include "caffe/data_transformer.hpp"
+#include "caffe/layers/data_layer.hpp"
#include "caffe/util/benchmark.hpp"
-#include "caffe/util/io.hpp"
namespace caffe {
diff --git a/src/caffe/layers/deconv_layer.cpp b/src/caffe/layers/deconv_layer.cpp
index 91aabb3..275c056 100644
--- a/src/caffe/layers/deconv_layer.cpp
+++ b/src/caffe/layers/deconv_layer.cpp
@@ -1,10 +1,6 @@
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/deconv_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/deconv_layer.cu b/src/caffe/layers/deconv_layer.cu
index 5dbdcc3..2267632 100644
--- a/src/caffe/layers/deconv_layer.cu
+++ b/src/caffe/layers/deconv_layer.cu
@@ -1,10 +1,6 @@
#include <vector>
-#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/im2col.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/deconv_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/dropout_layer.cpp b/src/caffe/layers/dropout_layer.cpp
index ec1256f..9cb64d9 100644
--- a/src/caffe/layers/dropout_layer.cpp
+++ b/src/caffe/layers/dropout_layer.cpp
@@ -2,11 +2,8 @@
#include <vector>
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/syncedmem.hpp"
+#include "caffe/layers/dropout_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/dropout_layer.cu b/src/caffe/layers/dropout_layer.cu
index f9ea04f..186c10c 100644
--- a/src/caffe/layers/dropout_layer.cu
+++ b/src/caffe/layers/dropout_layer.cu
@@ -1,16 +1,10 @@
-#include <algorithm>
-#include <limits>
#include <vector>
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/syncedmem.hpp"
+#include "caffe/layers/dropout_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
-
template <typename Dtype>
__global__ void DropoutForward(const int n, const Dtype* in,
const unsigned int* mask, const unsigned int threshold, const float scale,
@@ -73,5 +67,4 @@ void DropoutLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
INSTANTIATE_LAYER_GPU_FUNCS(DropoutLayer);
-
} // namespace caffe
diff --git a/src/caffe/layers/dummy_data_layer.cpp b/src/caffe/layers/dummy_data_layer.cpp
index 6b0d617..e382bfe 100644
--- a/src/caffe/layers/dummy_data_layer.cpp
+++ b/src/caffe/layers/dummy_data_layer.cpp
@@ -1,8 +1,7 @@
#include <vector>
#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/dummy_data_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/eltwise_layer.cpp b/src/caffe/layers/eltwise_layer.cpp
index a807007..2125616 100644
--- a/src/caffe/layers/eltwise_layer.cpp
+++ b/src/caffe/layers/eltwise_layer.cpp
@@ -1,9 +1,8 @@
#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/eltwise_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/eltwise_layer.cu b/src/caffe/layers/eltwise_layer.cu
index 2247870..c142852 100644
--- a/src/caffe/layers/eltwise_layer.cu
+++ b/src/caffe/layers/eltwise_layer.cu
@@ -1,9 +1,8 @@
#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/eltwise_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/embed_layer.cpp b/src/caffe/layers/embed_layer.cpp
index be6b2cd..36b40d7 100644
--- a/src/caffe/layers/embed_layer.cpp
+++ b/src/caffe/layers/embed_layer.cpp
@@ -1,10 +1,7 @@
#include <vector>
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/common_layers.hpp"
#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/embed_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
diff --git a/src/caffe/layers/embed_layer.cu b/src/caffe/layers/embed_layer.cu
index 62a4db8..6324a3a 100644
--- a/src/caffe/layers/embed_layer.cu
+++ b/src/caffe/layers/embed_layer.cu
@@ -1,10 +1,7 @@
#include <vector>
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/common_layers.hpp"
#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/embed_layer.hpp"
#include "caffe/util/gpu_util.cuh"
#include "caffe/util/math_functions.hpp"
diff --git a/src/caffe/layers/euclidean_loss_layer.cpp b/src/caffe/layers/euclidean_loss_layer.cpp
index 80efa31..300d991 100644
--- a/src/caffe/layers/euclidean_loss_layer.cpp
+++ b/src/caffe/layers/euclidean_loss_layer.cpp
@@ -1,9 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/euclidean_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/euclidean_loss_layer.cu b/src/caffe/layers/euclidean_loss_layer.cu
index 5b1de3a..4c221b6 100644
--- a/src/caffe/layers/euclidean_loss_layer.cu
+++ b/src/caffe/layers/euclidean_loss_layer.cu
@@ -1,9 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/euclidean_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/exp_layer.cpp b/src/caffe/layers/exp_layer.cpp
index c7e7c60..1f4a309 100644
--- a/src/caffe/layers/exp_layer.cpp
+++ b/src/caffe/layers/exp_layer.cpp
@@ -1,9 +1,7 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/exp_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/exp_layer.cu b/src/caffe/layers/exp_layer.cu
index 2d75d8d..61f7f11 100644
--- a/src/caffe/layers/exp_layer.cu
+++ b/src/caffe/layers/exp_layer.cu
@@ -1,9 +1,7 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/exp_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/filter_layer.cpp b/src/caffe/layers/filter_layer.cpp
index be1db32..e226c0b 100644
--- a/src/caffe/layers/filter_layer.cpp
+++ b/src/caffe/layers/filter_layer.cpp
@@ -1,9 +1,7 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/filter_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/filter_layer.cu b/src/caffe/layers/filter_layer.cu
index cf929ee..b01b16f 100644
--- a/src/caffe/layers/filter_layer.cu
+++ b/src/caffe/layers/filter_layer.cu
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/filter_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/flatten_layer.cpp b/src/caffe/layers/flatten_layer.cpp
index f7e5c9c..651507e 100644
--- a/src/caffe/layers/flatten_layer.cpp
+++ b/src/caffe/layers/flatten_layer.cpp
@@ -1,8 +1,6 @@
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/flatten_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/hdf5_data_layer.cpp b/src/caffe/layers/hdf5_data_layer.cpp
index 8ced510..2f13dc6 100644
--- a/src/caffe/layers/hdf5_data_layer.cpp
+++ b/src/caffe/layers/hdf5_data_layer.cpp
@@ -14,8 +14,7 @@ TODO:
#include "hdf5_hl.h"
#include "stdint.h"
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/hdf5_data_layer.hpp"
#include "caffe/util/hdf5.hpp"
namespace caffe {
diff --git a/src/caffe/layers/hdf5_data_layer.cu b/src/caffe/layers/hdf5_data_layer.cu
index 5e3e4ce..595d223 100644
--- a/src/caffe/layers/hdf5_data_layer.cu
+++ b/src/caffe/layers/hdf5_data_layer.cu
@@ -4,15 +4,12 @@ TODO:
*/
#include <stdint.h>
-#include <string>
#include <vector>
#include "hdf5.h"
#include "hdf5_hl.h"
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/hdf5_data_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/hdf5_output_layer.cpp b/src/caffe/layers/hdf5_output_layer.cpp
index 56788c2..f8f1edc 100644
--- a/src/caffe/layers/hdf5_output_layer.cpp
+++ b/src/caffe/layers/hdf5_output_layer.cpp
@@ -3,11 +3,8 @@
#include "hdf5.h"
#include "hdf5_hl.h"
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/hdf5_output_layer.hpp"
#include "caffe/util/hdf5.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/hdf5_output_layer.cu b/src/caffe/layers/hdf5_output_layer.cu
index eb6d0e4..c1685cd 100644
--- a/src/caffe/layers/hdf5_output_layer.cu
+++ b/src/caffe/layers/hdf5_output_layer.cu
@@ -3,10 +3,7 @@
#include "hdf5.h"
#include "hdf5_hl.h"
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/hdf5_output_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/hinge_loss_layer.cpp b/src/caffe/layers/hinge_loss_layer.cpp
index a2fb2a1..374aed3 100644
--- a/src/caffe/layers/hinge_loss_layer.cpp
+++ b/src/caffe/layers/hinge_loss_layer.cpp
@@ -1,12 +1,8 @@
#include <algorithm>
-#include <cfloat>
-#include <cmath>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/hinge_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/im2col_layer.cpp b/src/caffe/layers/im2col_layer.cpp
index 595c9db..c12e4f5 100644
--- a/src/caffe/layers/im2col_layer.cpp
+++ b/src/caffe/layers/im2col_layer.cpp
@@ -1,9 +1,7 @@
#include <vector>
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/im2col_layer.hpp"
#include "caffe/util/im2col.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/im2col_layer.cu b/src/caffe/layers/im2col_layer.cu
index cd50762..517b422 100644
--- a/src/caffe/layers/im2col_layer.cu
+++ b/src/caffe/layers/im2col_layer.cu
@@ -1,9 +1,7 @@
#include <vector>
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/im2col_layer.hpp"
#include "caffe/util/im2col.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/image_data_layer.cpp b/src/caffe/layers/image_data_layer.cpp
index 3d2190f..62fda4a 100644
--- a/src/caffe/layers/image_data_layer.cpp
+++ b/src/caffe/layers/image_data_layer.cpp
@@ -7,8 +7,9 @@
#include <utility>
#include <vector>
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/data_transformer.hpp"
+#include "caffe/layers/base_data_layer.hpp"
+#include "caffe/layers/image_data_layer.hpp"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/math_functions.hpp"
diff --git a/src/caffe/layers/infogain_loss_layer.cpp b/src/caffe/layers/infogain_loss_layer.cpp
index a1e0b40..624d311 100644
--- a/src/caffe/layers/infogain_loss_layer.cpp
+++ b/src/caffe/layers/infogain_loss_layer.cpp
@@ -1,12 +1,9 @@
#include <algorithm>
-#include <cfloat>
#include <cmath>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/infogain_loss_layer.hpp"
#include "caffe/util/io.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/inner_product_layer.cpp b/src/caffe/layers/inner_product_layer.cpp
index 83c3235..d908880 100644
--- a/src/caffe/layers/inner_product_layer.cpp
+++ b/src/caffe/layers/inner_product_layer.cpp
@@ -1,11 +1,8 @@
#include <vector>
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/inner_product_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/inner_product_layer.cu b/src/caffe/layers/inner_product_layer.cu
index c0ebd2c..dc25aa3 100644
--- a/src/caffe/layers/inner_product_layer.cu
+++ b/src/caffe/layers/inner_product_layer.cu
@@ -1,11 +1,8 @@
#include <vector>
-#include "caffe/blob.hpp"
-#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/inner_product_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/log_layer.cpp b/src/caffe/layers/log_layer.cpp
index 55a227f..c70a795 100644
--- a/src/caffe/layers/log_layer.cpp
+++ b/src/caffe/layers/log_layer.cpp
@@ -1,8 +1,6 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/neuron_layers.hpp"
+#include "caffe/layers/log_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
diff --git a/src/caffe/layers/log_layer.cu b/src/caffe/layers/log_layer.cu
index 847c86c..db466db 100644
--- a/src/caffe/layers/log_layer.cu
+++ b/src/caffe/layers/log_layer.cu
@@ -1,8 +1,6 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/neuron_layers.hpp"
+#include "caffe/layers/log_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
diff --git a/src/caffe/layers/loss_layer.cpp b/src/caffe/layers/loss_layer.cpp
index 3496a5c..c0b7a86 100644
--- a/src/caffe/layers/loss_layer.cpp
+++ b/src/caffe/layers/loss_layer.cpp
@@ -1,12 +1,6 @@
-#include <algorithm>
-#include <cfloat>
-#include <cmath>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/loss_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/lrn_layer.cpp b/src/caffe/layers/lrn_layer.cpp
index 36c1ace..210525e 100644
--- a/src/caffe/layers/lrn_layer.cpp
+++ b/src/caffe/layers/lrn_layer.cpp
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/lrn_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
@@ -254,6 +253,5 @@ STUB_GPU_BACKWARD(LRNLayer, CrossChannelBackward);
#endif
INSTANTIATE_CLASS(LRNLayer);
-REGISTER_LAYER_CLASS(LRN);
} // namespace caffe
diff --git a/src/caffe/layers/lrn_layer.cu b/src/caffe/layers/lrn_layer.cu
index 001b3c3..26e619c 100644
--- a/src/caffe/layers/lrn_layer.cu
+++ b/src/caffe/layers/lrn_layer.cu
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/lrn_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/memory_data_layer.cpp b/src/caffe/layers/memory_data_layer.cpp
index 2370aa0..8290987 100644
--- a/src/caffe/layers/memory_data_layer.cpp
+++ b/src/caffe/layers/memory_data_layer.cpp
@@ -4,9 +4,7 @@
#include <vector>
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/memory_data_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/multinomial_logistic_loss_layer.cpp b/src/caffe/layers/multinomial_logistic_loss_layer.cpp
index 4267a59..6566499 100644
--- a/src/caffe/layers/multinomial_logistic_loss_layer.cpp
+++ b/src/caffe/layers/multinomial_logistic_loss_layer.cpp
@@ -1,12 +1,9 @@
#include <algorithm>
-#include <cfloat>
#include <cmath>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/util/io.hpp"
+#include "caffe/layers/multinomial_logistic_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/mvn_layer.cpp b/src/caffe/layers/mvn_layer.cpp
index 325691b..8fe4ef8 100644
--- a/src/caffe/layers/mvn_layer.cpp
+++ b/src/caffe/layers/mvn_layer.cpp
@@ -1,8 +1,6 @@
-#include <algorithm>
#include <vector>
-#include "caffe/common_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/mvn_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
@@ -42,29 +40,21 @@ void MVNLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
int dim = bottom[0]->count() / num;
- if (this->layer_param_.mvn_param().normalize_variance()) {
- // put the squares of bottom into temp_
- caffe_powx(bottom[0]->count(), bottom_data, Dtype(2),
- temp_.mutable_cpu_data());
+ // subtract mean
+ caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, bottom_data,
+ sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX
+ caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, -1.,
+ mean_.cpu_data(), sum_multiplier_.cpu_data(), 0.,
+ temp_.mutable_cpu_data());
+ caffe_add(temp_.count(), bottom_data, temp_.cpu_data(), top_data); // X-EX
- // computes variance using var(X) = E(X^2) - (EX)^2
- caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, bottom_data,
- sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX
+ if (this->layer_param_.mvn_param().normalize_variance()) {
+ // compute variance using var(X) = E((X-EX)^2)
+ caffe_powx(bottom[0]->count(), top_data, Dtype(2),
+ temp_.mutable_cpu_data()); // (X-EX)^2
caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, temp_.cpu_data(),
sum_multiplier_.cpu_data(), 0.,
- variance_.mutable_cpu_data()); // E(X^2)
- caffe_powx(mean_.count(), mean_.cpu_data(), Dtype(2),
- temp_.mutable_cpu_data()); // (EX)^2
- caffe_sub(mean_.count(), variance_.cpu_data(), temp_.cpu_data(),
- variance_.mutable_cpu_data()); // variance
-
- // do mean and variance normalization
- // subtract mean
- caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, -1.,
- mean_.cpu_data(), sum_multiplier_.cpu_data(), 0.,
- temp_.mutable_cpu_data());
-
- caffe_add(temp_.count(), bottom_data, temp_.cpu_data(), top_data);
+ variance_.mutable_cpu_data()); // E((X-EX)^2)
// normalize variance
caffe_powx(variance_.count(), variance_.cpu_data(), Dtype(0.5),
@@ -77,16 +67,6 @@ void MVNLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
temp_.mutable_cpu_data());
caffe_div(temp_.count(), top_data, temp_.cpu_data(), top_data);
- } else {
- caffe_cpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, bottom_data,
- sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX
-
- // subtract mean
- caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, -1.,
- mean_.cpu_data(), sum_multiplier_.cpu_data(), 0.,
- temp_.mutable_cpu_data());
-
- caffe_add(temp_.count(), bottom_data, temp_.cpu_data(), top_data);
}
}
diff --git a/src/caffe/layers/mvn_layer.cu b/src/caffe/layers/mvn_layer.cu
index d86a2e7..739293b 100644
--- a/src/caffe/layers/mvn_layer.cu
+++ b/src/caffe/layers/mvn_layer.cu
@@ -1,8 +1,6 @@
-#include <algorithm>
#include <vector>
-#include "caffe/common_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/mvn_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
@@ -20,29 +18,22 @@ void MVNLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
int dim = bottom[0]->count() / num;
- if (this->layer_param_.mvn_param().normalize_variance()) {
- // put the squares of bottom into temp_
- caffe_gpu_powx(bottom[0]->count(), bottom_data, Dtype(2),
- temp_.mutable_gpu_data());
+ // subtract mean
+ caffe_gpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, bottom_data,
+ sum_multiplier_.gpu_data(), 0., mean_.mutable_gpu_data()); // EX
+ caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, -1.,
+ mean_.gpu_data(), sum_multiplier_.gpu_data(), 0.,
+ temp_.mutable_gpu_data());
+ caffe_gpu_add(temp_.count(), bottom_data, temp_.gpu_data(),
+ top_data); // X-EX
- // computes variance using var(X) = E(X^2) - (EX)^2
- caffe_gpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, bottom_data,
- sum_multiplier_.gpu_data(), 0., mean_.mutable_gpu_data()); // EX
+ if (this->layer_param_.mvn_param().normalize_variance()) {
+ // compute variance using var(X) = E((X-EX)^2)
+ caffe_gpu_powx(bottom[0]->count(), top_data, Dtype(2),
+ temp_.mutable_gpu_data()); // (X-EX)^2
caffe_gpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, temp_.gpu_data(),
sum_multiplier_.gpu_data(), 0.,
- variance_.mutable_gpu_data()); // E(X^2)
- caffe_gpu_powx(mean_.count(), mean_.gpu_data(), Dtype(2),
- temp_.mutable_gpu_data()); // (EX)^2
- caffe_gpu_sub(mean_.count(), variance_.gpu_data(), temp_.gpu_data(),
- variance_.mutable_gpu_data()); // variance
-
- // do mean and variance normalization
- // subtract mean
- caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, -1.,
- mean_.gpu_data(), sum_multiplier_.gpu_data(), 0.,
- temp_.mutable_gpu_data());
-
- caffe_gpu_add(temp_.count(), bottom_data, temp_.gpu_data(), top_data);
+ variance_.mutable_gpu_data()); // E((X-EX)^2)
// normalize variance
caffe_gpu_powx(variance_.count(), variance_.gpu_data(), Dtype(0.5),
@@ -55,16 +46,6 @@ void MVNLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
temp_.mutable_gpu_data());
caffe_gpu_div(temp_.count(), top_data, temp_.gpu_data(), top_data);
- } else {
- caffe_gpu_gemv<Dtype>(CblasNoTrans, num, dim, 1. / dim, bottom_data,
- sum_multiplier_.gpu_data(), 0., mean_.mutable_gpu_data()); // EX
-
- // subtract mean
- caffe_gpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, dim, 1, -1.,
- mean_.gpu_data(), sum_multiplier_.gpu_data(), 0.,
- temp_.mutable_gpu_data());
-
- caffe_gpu_add(temp_.count(), bottom_data, temp_.gpu_data(), top_data);
}
}
diff --git a/src/caffe/layers/neuron_layer.cpp b/src/caffe/layers/neuron_layer.cpp
index ba67b43..d7b5f38 100644
--- a/src/caffe/layers/neuron_layer.cpp
+++ b/src/caffe/layers/neuron_layer.cpp
@@ -1,7 +1,6 @@
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/neuron_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/pooling_layer.cpp b/src/caffe/layers/pooling_layer.cpp
index c8d4149..90897db 100644
--- a/src/caffe/layers/pooling_layer.cpp
+++ b/src/caffe/layers/pooling_layer.cpp
@@ -2,11 +2,8 @@
#include <cfloat>
#include <vector>
-#include "caffe/common.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/syncedmem.hpp"
+#include "caffe/layers/pooling_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/pooling_layer.cu b/src/caffe/layers/pooling_layer.cu
index ca4b13f..1ea46cc 100644
--- a/src/caffe/layers/pooling_layer.cu
+++ b/src/caffe/layers/pooling_layer.cu
@@ -2,9 +2,8 @@
#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/pooling_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/power_layer.cpp b/src/caffe/layers/power_layer.cpp
index 4fe34c4..d99b77c 100644
--- a/src/caffe/layers/power_layer.cpp
+++ b/src/caffe/layers/power_layer.cpp
@@ -1,9 +1,7 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/power_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/power_layer.cu b/src/caffe/layers/power_layer.cu
index 90d9440..07711c4 100644
--- a/src/caffe/layers/power_layer.cu
+++ b/src/caffe/layers/power_layer.cu
@@ -1,9 +1,7 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/power_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/prelu_layer.cpp b/src/caffe/layers/prelu_layer.cpp
index 8183175..853181b 100644
--- a/src/caffe/layers/prelu_layer.cpp
+++ b/src/caffe/layers/prelu_layer.cpp
@@ -2,8 +2,9 @@
#include <vector>
#include "caffe/filler.hpp"
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+
+#include "caffe/layers/neuron_layer.hpp"
+#include "caffe/layers/prelu_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/prelu_layer.cu b/src/caffe/layers/prelu_layer.cu
index e1f2004..aeb80ea 100644
--- a/src/caffe/layers/prelu_layer.cu
+++ b/src/caffe/layers/prelu_layer.cu
@@ -1,8 +1,8 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/neuron_layer.hpp"
+#include "caffe/layers/prelu_layer.hpp"
namespace caffe {
@@ -31,10 +31,15 @@ __global__ void PReLUBackward(const int n, const int channels, const int dim,
// CUDA kernel for element-wise parameter backward
template <typename Dtype>
-__global__ void PReLUParamBackward(const int n, const Dtype* in_diff,
+__global__ void PReLUParamBackward(const int n,
+ const int rows, const int rowPitch, const Dtype* in_diff,
const Dtype* in_data, Dtype* out_diff) {
CUDA_KERNEL_LOOP(index, n) {
out_diff[index] = in_diff[index] * in_data[index] * (in_data[index] <= 0);
+ for ( int k = 1; k < rows; k++ ) {
+ out_diff[index] += in_diff[index + k*rowPitch]
+ * in_data[index + k*rowPitch] * (in_data[index + k*rowPitch] <= 0);
+ }
}
}
@@ -82,29 +87,24 @@ void PReLULayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
if (this->param_propagate_down_[0]) {
Dtype* slope_diff = this->blobs_[0]->mutable_gpu_diff();
int cdim = channels * dim;
- Dtype dsum = 0.;
- for (int n = 0; n < bottom[0]->num(); ++n) {
- // compute element-wise diff
- // NOLINT_NEXT_LINE(whitespace/operators)
- PReLUParamBackward<Dtype><<<CAFFE_GET_BLOCKS(cdim),
- CAFFE_CUDA_NUM_THREADS>>>(
- cdim, top_diff + top[0]->offset(n),
- bottom_data + bottom[0]->offset(n),
- backward_buff_.mutable_gpu_diff());
- CUDA_POST_KERNEL_CHECK;
- if (channel_shared_) {
- Dtype d;
- caffe_gpu_dot<Dtype>(channels * dim, backward_buff_.gpu_diff(),
- multiplier_.gpu_data(), &d);
- dsum += d;
- } else {
- caffe_gpu_gemv<Dtype>(CblasNoTrans, channels, dim, 1.,
- backward_buff_.gpu_diff(), multiplier_.gpu_data(), 1.,
- slope_diff);
- }
- }
+
+ // compute element-wise diff
+ // NOLINT_NEXT_LINE(whitespace/operators)
+ PReLUParamBackward<Dtype><<<CAFFE_GET_BLOCKS(cdim),
+ CAFFE_CUDA_NUM_THREADS>>>(
+ cdim, bottom[0]->num(), top[0]->offset(1), top_diff ,
+ bottom_data ,
+ backward_buff_.mutable_gpu_diff());
+ CUDA_POST_KERNEL_CHECK;
if (channel_shared_) {
+ Dtype dsum;
+ caffe_gpu_dot<Dtype>(channels * dim, backward_buff_.gpu_diff(),
+ multiplier_.gpu_data(), &dsum);
caffe_gpu_add_scalar(this->blobs_[0]->count(), Dtype(dsum), slope_diff);
+ } else {
+ caffe_gpu_gemv<Dtype>(CblasNoTrans, channels, dim, 1.,
+ backward_buff_.gpu_diff(), multiplier_.gpu_data(), 1.,
+ slope_diff);
}
}
// Propagate to bottom
diff --git a/src/caffe/layers/reduction_layer.cpp b/src/caffe/layers/reduction_layer.cpp
index 8ae6329..fa46487 100644
--- a/src/caffe/layers/reduction_layer.cpp
+++ b/src/caffe/layers/reduction_layer.cpp
@@ -1,10 +1,7 @@
-#include <algorithm>
-#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/reduction_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/reduction_layer.cu b/src/caffe/layers/reduction_layer.cu
index 2dbd3bc..4a6b2b7 100644
--- a/src/caffe/layers/reduction_layer.cu
+++ b/src/caffe/layers/reduction_layer.cu
@@ -1,9 +1,7 @@
-#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/reduction_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/relu_layer.cpp b/src/caffe/layers/relu_layer.cpp
index cc00319..92a729c 100644
--- a/src/caffe/layers/relu_layer.cpp
+++ b/src/caffe/layers/relu_layer.cpp
@@ -1,8 +1,7 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/relu_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/relu_layer.cu b/src/caffe/layers/relu_layer.cu
index b8924c8..4bf15b3 100644
--- a/src/caffe/layers/relu_layer.cu
+++ b/src/caffe/layers/relu_layer.cu
@@ -1,8 +1,7 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/relu_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/reshape_layer.cpp b/src/caffe/layers/reshape_layer.cpp
index ffe970f..82339f7 100644
--- a/src/caffe/layers/reshape_layer.cpp
+++ b/src/caffe/layers/reshape_layer.cpp
@@ -1,7 +1,6 @@
#include <vector>
-#include "caffe/common_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/reshape_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp
index cc236fe..10ac947 100644
--- a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp
+++ b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp
@@ -1,10 +1,7 @@
-#include <algorithm>
-#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/sigmoid_cross_entropy_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu
index 547fa80..046cb9d 100644
--- a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu
+++ b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu
@@ -1,10 +1,7 @@
-#include <algorithm>
-#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/sigmoid_cross_entropy_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/sigmoid_layer.cpp b/src/caffe/layers/sigmoid_layer.cpp
index 48c3849..85fd967 100644
--- a/src/caffe/layers/sigmoid_layer.cpp
+++ b/src/caffe/layers/sigmoid_layer.cpp
@@ -1,9 +1,7 @@
-#include <algorithm>
#include <cmath>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/sigmoid_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/sigmoid_layer.cu b/src/caffe/layers/sigmoid_layer.cu
index e1af065..184c61e 100644
--- a/src/caffe/layers/sigmoid_layer.cu
+++ b/src/caffe/layers/sigmoid_layer.cu
@@ -1,9 +1,7 @@
-#include <algorithm>
#include <cmath>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/sigmoid_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/silence_layer.cpp b/src/caffe/layers/silence_layer.cpp
index 4abf9ef..b2f85c5 100644
--- a/src/caffe/layers/silence_layer.cpp
+++ b/src/caffe/layers/silence_layer.cpp
@@ -1,7 +1,6 @@
#include <vector>
-#include "caffe/common_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/silence_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
@@ -12,7 +11,7 @@ void SilenceLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
for (int i = 0; i < bottom.size(); ++i) {
if (propagate_down[i]) {
caffe_set(bottom[i]->count(), Dtype(0),
- bottom[i]->mutable_cpu_data());
+ bottom[i]->mutable_cpu_diff());
}
}
}
diff --git a/src/caffe/layers/silence_layer.cu b/src/caffe/layers/silence_layer.cu
index 8d044ee..3494f6f 100644
--- a/src/caffe/layers/silence_layer.cu
+++ b/src/caffe/layers/silence_layer.cu
@@ -1,7 +1,6 @@
#include <vector>
-#include "caffe/common_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/silence_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
@@ -18,7 +17,7 @@ void SilenceLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
for (int i = 0; i < bottom.size(); ++i) {
if (propagate_down[i]) {
caffe_gpu_set(bottom[i]->count(), Dtype(0),
- bottom[i]->mutable_gpu_data());
+ bottom[i]->mutable_gpu_diff());
}
}
}
diff --git a/src/caffe/layers/slice_layer.cpp b/src/caffe/layers/slice_layer.cpp
index 0a059ae..759beaf 100644
--- a/src/caffe/layers/slice_layer.cpp
+++ b/src/caffe/layers/slice_layer.cpp
@@ -1,9 +1,8 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/slice_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/slice_layer.cu b/src/caffe/layers/slice_layer.cu
index e8dc6cd..1be3a79 100644
--- a/src/caffe/layers/slice_layer.cu
+++ b/src/caffe/layers/slice_layer.cu
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/slice_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/softmax_layer.cpp b/src/caffe/layers/softmax_layer.cpp
index 04712c9..f60e9b0 100644
--- a/src/caffe/layers/softmax_layer.cpp
+++ b/src/caffe/layers/softmax_layer.cpp
@@ -1,9 +1,8 @@
#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/softmax_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/softmax_layer.cu b/src/caffe/layers/softmax_layer.cu
index 1f9c3a4..7a9e683 100644
--- a/src/caffe/layers/softmax_layer.cu
+++ b/src/caffe/layers/softmax_layer.cu
@@ -4,9 +4,8 @@
#include "thrust/device_vector.h"
-#include "caffe/layer.hpp"
+#include "caffe/layers/softmax_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/softmax_loss_layer.cpp b/src/caffe/layers/softmax_loss_layer.cpp
index ba312f6..dddb760 100644
--- a/src/caffe/layers/softmax_loss_layer.cpp
+++ b/src/caffe/layers/softmax_loss_layer.cpp
@@ -2,10 +2,8 @@
#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/layer_factory.hpp"
+#include "caffe/layers/softmax_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
@@ -27,7 +25,14 @@ void SoftmaxWithLossLayer<Dtype>::LayerSetUp(
if (has_ignore_label_) {
ignore_label_ = this->layer_param_.loss_param().ignore_label();
}
- normalize_ = this->layer_param_.loss_param().normalize();
+ if (!this->layer_param_.loss_param().has_normalization() &&
+ this->layer_param_.loss_param().has_normalize()) {
+ normalization_ = this->layer_param_.loss_param().normalize() ?
+ LossParameter_NormalizationMode_VALID :
+ LossParameter_NormalizationMode_BATCH_SIZE;
+ } else {
+ normalization_ = this->layer_param_.loss_param().normalization();
+ }
}
template <typename Dtype>
@@ -51,6 +56,36 @@ void SoftmaxWithLossLayer<Dtype>::Reshape(
}
template <typename Dtype>
+Dtype SoftmaxWithLossLayer<Dtype>::get_normalizer(
+ LossParameter_NormalizationMode normalization_mode, int valid_count) {
+ Dtype normalizer;
+ switch (normalization_mode) {
+ case LossParameter_NormalizationMode_FULL:
+ normalizer = Dtype(outer_num_ * inner_num_);
+ break;
+ case LossParameter_NormalizationMode_VALID:
+ if (valid_count == -1) {
+ normalizer = Dtype(outer_num_ * inner_num_);
+ } else {
+ normalizer = Dtype(valid_count);
+ }
+ break;
+ case LossParameter_NormalizationMode_BATCH_SIZE:
+ normalizer = Dtype(outer_num_);
+ break;
+ case LossParameter_NormalizationMode_NONE:
+ normalizer = Dtype(1);
+ break;
+ default:
+ LOG(FATAL) << "Unknown normalization mode: "
+ << LossParameter_NormalizationMode_Name(normalization_mode);
+ }
+ // Some users will have no labels for some examples in order to 'turn off' a
+ // particular loss in a multi-task setup. The max prevents NaNs in that case.
+ return std::max(Dtype(1.0), normalizer);
+}
+
+template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Forward_cpu(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
// The forward pass computes the softmax prob values.
@@ -73,11 +108,7 @@ void SoftmaxWithLossLayer<Dtype>::Forward_cpu(
++count;
}
}
- if (normalize_) {
- top[0]->mutable_cpu_data()[0] = loss / count;
- } else {
- top[0]->mutable_cpu_data()[0] = loss / outer_num_;
- }
+ top[0]->mutable_cpu_data()[0] = loss / get_normalizer(normalization_, count);
if (top.size() == 2) {
top[1]->ShareData(prob_);
}
@@ -111,12 +142,9 @@ void SoftmaxWithLossLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
}
}
// Scale gradient
- const Dtype loss_weight = top[0]->cpu_diff()[0];
- if (normalize_) {
- caffe_scal(prob_.count(), loss_weight / count, bottom_diff);
- } else {
- caffe_scal(prob_.count(), loss_weight / outer_num_, bottom_diff);
- }
+ Dtype loss_weight = top[0]->cpu_diff()[0] /
+ get_normalizer(normalization_, count);
+ caffe_scal(prob_.count(), loss_weight, bottom_diff);
}
}
diff --git a/src/caffe/layers/softmax_loss_layer.cu b/src/caffe/layers/softmax_loss_layer.cu
index 7e0f3da..660e1b3 100644
--- a/src/caffe/layers/softmax_loss_layer.cu
+++ b/src/caffe/layers/softmax_loss_layer.cu
@@ -2,9 +2,8 @@
#include <cfloat>
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/softmax_loss_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
@@ -50,14 +49,15 @@ void SoftmaxWithLossLayer<Dtype>::Forward_gpu(
outer_num_, dim, inner_num_, has_ignore_label_, ignore_label_, counts);
Dtype loss;
caffe_gpu_asum(nthreads, loss_data, &loss);
- if (normalize_) {
- Dtype count;
- caffe_gpu_asum(nthreads, counts, &count);
- loss /= count;
- } else {
- loss /= outer_num_;
+ Dtype valid_count = -1;
+ // Only launch another CUDA kernel if we actually need the count of valid
+ // outputs.
+ if (normalization_ == LossParameter_NormalizationMode_VALID &&
+ has_ignore_label_) {
+ caffe_gpu_asum(nthreads, counts, &valid_count);
}
- top[0]->mutable_cpu_data()[0] = loss;
+ top[0]->mutable_cpu_data()[0] = loss / get_normalizer(normalization_,
+ valid_count);
if (top.size() == 2) {
top[1]->ShareData(prob_);
}
@@ -109,14 +109,17 @@ void SoftmaxWithLossLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
SoftmaxLossBackwardGPU<Dtype><<<CAFFE_GET_BLOCKS(nthreads),
CAFFE_CUDA_NUM_THREADS>>>(nthreads, top_data, label, bottom_diff,
outer_num_, dim, inner_num_, has_ignore_label_, ignore_label_, counts);
- const Dtype loss_weight = top[0]->cpu_diff()[0];
- if (normalize_) {
- Dtype count;
- caffe_gpu_asum(nthreads, counts, &count);
- caffe_gpu_scal(prob_.count(), loss_weight / count, bottom_diff);
- } else {
- caffe_gpu_scal(prob_.count(), loss_weight / outer_num_, bottom_diff);
+
+ Dtype valid_count = -1;
+ // Only launch another CUDA kernel if we actually need the count of valid
+ // outputs.
+ if (normalization_ == LossParameter_NormalizationMode_VALID &&
+ has_ignore_label_) {
+ caffe_gpu_asum(nthreads, counts, &valid_count);
}
+ const Dtype loss_weight = top[0]->cpu_diff()[0] /
+ get_normalizer(normalization_, valid_count);
+ caffe_gpu_scal(prob_.count(), loss_weight , bottom_diff);
}
}
diff --git a/src/caffe/layers/split_layer.cpp b/src/caffe/layers/split_layer.cpp
index 272cb59..1a27a9a 100644
--- a/src/caffe/layers/split_layer.cpp
+++ b/src/caffe/layers/split_layer.cpp
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/split_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/split_layer.cu b/src/caffe/layers/split_layer.cu
index a4f5df2..bec9987 100644
--- a/src/caffe/layers/split_layer.cu
+++ b/src/caffe/layers/split_layer.cu
@@ -1,8 +1,7 @@
#include <vector>
-#include "caffe/layer.hpp"
+#include "caffe/layers/split_layer.hpp"
#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
namespace caffe {
diff --git a/src/caffe/layers/spp_layer.cpp b/src/caffe/layers/spp_layer.cpp
index d762291..b9af8e8 100644
--- a/src/caffe/layers/spp_layer.cpp
+++ b/src/caffe/layers/spp_layer.cpp
@@ -1,12 +1,12 @@
#include <algorithm>
-#include <cfloat>
#include <vector>
-#include "caffe/common.hpp"
#include "caffe/layer.hpp"
-#include "caffe/syncedmem.hpp"
-#include "caffe/util/math_functions.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/concat_layer.hpp"
+#include "caffe/layers/flatten_layer.hpp"
+#include "caffe/layers/pooling_layer.hpp"
+#include "caffe/layers/split_layer.hpp"
+#include "caffe/layers/spp_layer.hpp"
namespace caffe {
@@ -222,7 +222,6 @@ void SPPLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
split_layer_->Backward(split_top_vec_, propagate_down, bottom);
}
-
INSTANTIATE_CLASS(SPPLayer);
REGISTER_LAYER_CLASS(SPP);
diff --git a/src/caffe/layers/tanh_layer.cpp b/src/caffe/layers/tanh_layer.cpp
index ee5ed77..184e926 100644
--- a/src/caffe/layers/tanh_layer.cpp
+++ b/src/caffe/layers/tanh_layer.cpp
@@ -1,11 +1,9 @@
// TanH neuron activation function layer.
// Adapted from ReLU layer code written by Yangqing Jia
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/tanh_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/tanh_layer.cu b/src/caffe/layers/tanh_layer.cu
index ccd6e63..cbfc178 100644
--- a/src/caffe/layers/tanh_layer.cu
+++ b/src/caffe/layers/tanh_layer.cu
@@ -1,11 +1,9 @@
// TanH neuron activation function layer.
// Adapted from ReLU layer code written by Yangqing Jia
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/tanh_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/threshold_layer.cpp b/src/caffe/layers/threshold_layer.cpp
index 2365e7b..63822ee 100644
--- a/src/caffe/layers/threshold_layer.cpp
+++ b/src/caffe/layers/threshold_layer.cpp
@@ -1,8 +1,6 @@
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
-
+#include "caffe/layers/threshold_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/threshold_layer.cu b/src/caffe/layers/threshold_layer.cu
index bfa7f15..b0b0665 100644
--- a/src/caffe/layers/threshold_layer.cu
+++ b/src/caffe/layers/threshold_layer.cu
@@ -1,8 +1,6 @@
-#include <algorithm>
#include <vector>
-#include "caffe/layer.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/threshold_layer.hpp"
namespace caffe {
diff --git a/src/caffe/layers/tile_layer.cpp b/src/caffe/layers/tile_layer.cpp
index f55008c..cf0c187 100644
--- a/src/caffe/layers/tile_layer.cpp
+++ b/src/caffe/layers/tile_layer.cpp
@@ -1,7 +1,6 @@
#include <vector>
-#include "caffe/common_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/tile_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
diff --git a/src/caffe/layers/tile_layer.cu b/src/caffe/layers/tile_layer.cu
index 7fd3bc4..282049e 100644
--- a/src/caffe/layers/tile_layer.cu
+++ b/src/caffe/layers/tile_layer.cu
@@ -1,7 +1,6 @@
#include <vector>
-#include "caffe/common_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/layers/tile_layer.hpp"
#include "caffe/util/math_functions.hpp"
namespace caffe {
diff --git a/src/caffe/layers/window_data_layer.cpp b/src/caffe/layers/window_data_layer.cpp
index f8db61c..4ca8315 100644
--- a/src/caffe/layers/window_data_layer.cpp
+++ b/src/caffe/layers/window_data_layer.cpp
@@ -12,9 +12,10 @@
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
-#include "caffe/common.hpp"
-#include "caffe/data_layers.hpp"
-#include "caffe/layer.hpp"
+#include "caffe/data_transformer.hpp"
+#include "caffe/internal_thread.hpp"
+#include "caffe/layers/base_data_layer.hpp"
+#include "caffe/layers/window_data_layer.hpp"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/math_functions.hpp"
diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp
index ebb8b5d..05bee79 100644
--- a/src/caffe/net.cpp
+++ b/src/caffe/net.cpp
@@ -46,10 +46,9 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
// the current NetState.
NetParameter filtered_param;
FilterNet(in_param, &filtered_param);
- if (Caffe::root_solver()) {
- LOG(INFO) << "Initializing net from parameters: " << std::endl
- << filtered_param.DebugString();
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "Initializing net from parameters: " << std::endl
+ << filtered_param.DebugString();
// Create a copy of filtered_param with splits added where necessary.
NetParameter param;
InsertSplits(filtered_param, ¶m);
@@ -73,8 +72,6 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
const int layer_id = -1; // inputs have fake layer ID -1
AppendTop(param, layer_id, input_id, &available_blobs, &blob_name_to_idx);
}
- DLOG_IF(INFO, Caffe::root_solver())
- << "Memory required for data: " << memory_used_ * sizeof(Dtype);
// For each layer, set up its input and output
bottom_vecs_.resize(param.layer_size());
top_vecs_.resize(param.layer_size());
@@ -106,9 +103,8 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
layers_.push_back(LayerRegistry<Dtype>::CreateLayer(layer_param));
}
layer_names_.push_back(layer_param.name());
- if (Caffe::root_solver()) {
- LOG(INFO) << "Creating Layer " << layer_param.name();
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "Creating Layer " << layer_param.name();
bool need_backward = false;
// Figure out this layer's input and output
@@ -151,29 +147,23 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
} else {
layers_[layer_id]->SetUp(bottom_vecs_[layer_id], top_vecs_[layer_id]);
}
- if (Caffe::root_solver()) {
- LOG(INFO) << "Setting up " << layer_names_[layer_id];
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "Setting up " << layer_names_[layer_id];
for (int top_id = 0; top_id < top_vecs_[layer_id].size(); ++top_id) {
if (blob_loss_weights_.size() <= top_id_vecs_[layer_id][top_id]) {
blob_loss_weights_.resize(top_id_vecs_[layer_id][top_id] + 1, Dtype(0));
}
blob_loss_weights_[top_id_vecs_[layer_id][top_id]] = layer->loss(top_id);
- if (Caffe::root_solver()) {
- LOG(INFO) << "Top shape: "
- << top_vecs_[layer_id][top_id]->shape_string();
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "Top shape: " << top_vecs_[layer_id][top_id]->shape_string();
if (layer->loss(top_id)) {
- if (Caffe::root_solver()) {
- LOG(INFO) << " with loss weight " << layer->loss(top_id);
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " with loss weight " << layer->loss(top_id);
}
memory_used_ += top_vecs_[layer_id][top_id]->count();
}
- if (Caffe::root_solver()) {
- DLOG(INFO) << "Memory required for data: "
- << memory_used_ * sizeof(Dtype);
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "Memory required for data: " << memory_used_ * sizeof(Dtype);
const int param_size = layer_param.param_size();
const int num_param_blobs = layers_[layer_id]->blobs().size();
CHECK_LE(param_size, num_param_blobs)
@@ -231,14 +221,12 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
}
}
if (!layer_contributes_loss) { layer_need_backward_[layer_id] = false; }
- if (layer_need_backward_[layer_id]) {
- if (Caffe::root_solver()) {
+ if (Caffe::root_solver()) {
+ if (layer_need_backward_[layer_id]) {
LOG(INFO) << layer_names_[layer_id] << " needs backward computation.";
- }
- } else {
- if (Caffe::root_solver()) {
+ } else {
LOG(INFO) << layer_names_[layer_id]
- << " does not need backward computation.";
+ << " does not need backward computation.";
}
}
for (int bottom_id = 0; bottom_id < bottom_vecs_[layer_id].size();
@@ -279,9 +267,8 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
// In the end, all remaining blobs are considered output blobs.
for (set<string>::iterator it = available_blobs.begin();
it != available_blobs.end(); ++it) {
- if (Caffe::root_solver()) {
- LOG(INFO) << "This network produces output " << *it;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "This network produces output " << *it;
net_output_blobs_.push_back(blobs_[blob_name_to_idx[*it]].get());
net_output_blob_indices_.push_back(blob_name_to_idx[*it]);
}
@@ -293,10 +280,7 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
}
ShareWeights();
debug_info_ = param.debug_info();
- if (Caffe::root_solver()) {
- LOG(INFO) << "Network initialization done.";
- LOG(INFO) << "Memory required for data: " << memory_used_ * sizeof(Dtype);
- }
+ LOG_IF(INFO, Caffe::root_solver()) << "Network initialization done.";
}
template <typename Dtype>
@@ -335,33 +319,30 @@ bool Net<Dtype>::StateMeetsRule(const NetState& state,
// Check whether the rule is broken due to phase.
if (rule.has_phase()) {
if (rule.phase() != state.phase()) {
- if (Caffe::root_solver()) {
- LOG(INFO) << "The NetState phase (" << state.phase()
- << ") differed from the phase (" << rule.phase()
- << ") specified by a rule in layer " << layer_name;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "The NetState phase (" << state.phase()
+ << ") differed from the phase (" << rule.phase()
+ << ") specified by a rule in layer " << layer_name;
return false;
}
}
// Check whether the rule is broken due to min level.
if (rule.has_min_level()) {
if (state.level() < rule.min_level()) {
- if (Caffe::root_solver()) {
- LOG(INFO) << "The NetState level (" << state.level()
- << ") is above the min_level (" << rule.min_level()
- << ") specified by a rule in layer " << layer_name;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "The NetState level (" << state.level()
+ << ") is above the min_level (" << rule.min_level()
+ << ") specified by a rule in layer " << layer_name;
return false;
}
}
// Check whether the rule is broken due to max level.
if (rule.has_max_level()) {
if (state.level() > rule.max_level()) {
- if (Caffe::root_solver()) {
- LOG(INFO) << "The NetState level (" << state.level()
- << ") is above the max_level (" << rule.max_level()
- << ") specified by a rule in layer " << layer_name;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "The NetState level (" << state.level()
+ << ") is above the max_level (" << rule.max_level()
+ << ") specified by a rule in layer " << layer_name;
return false;
}
}
@@ -374,10 +355,9 @@ bool Net<Dtype>::StateMeetsRule(const NetState& state,
if (rule.stage(i) == state.stage(j)) { has_stage = true; }
}
if (!has_stage) {
- if (Caffe::root_solver()) {
- LOG(INFO) << "The NetState did not contain stage '" << rule.stage(i)
- << "' specified by a rule in layer " << layer_name;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "The NetState did not contain stage '" << rule.stage(i)
+ << "' specified by a rule in layer " << layer_name;
return false;
}
}
@@ -390,10 +370,9 @@ bool Net<Dtype>::StateMeetsRule(const NetState& state,
if (rule.not_stage(i) == state.stage(j)) { has_stage = true; }
}
if (has_stage) {
- if (Caffe::root_solver()) {
- LOG(INFO) << "The NetState contained a not_stage '" << rule.not_stage(i)
- << "' specified by a rule in layer " << layer_name;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << "The NetState contained a not_stage '" << rule.not_stage(i)
+ << "' specified by a rule in layer " << layer_name;
return false;
}
}
@@ -415,9 +394,8 @@ void Net<Dtype>::AppendTop(const NetParameter& param, const int layer_id,
if (blob_name_to_idx && layer_param && layer_param->bottom_size() > top_id &&
blob_name == layer_param->bottom(top_id)) {
// In-place computation
- if (Caffe::root_solver()) {
- LOG(INFO) << layer_param->name() << " -> " << blob_name << " (in-place)";
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << layer_param->name() << " -> " << blob_name << " (in-place)";
top_vecs_[layer_id].push_back(blobs_[(*blob_name_to_idx)[blob_name]].get());
top_id_vecs_[layer_id].push_back((*blob_name_to_idx)[blob_name]);
} else if (blob_name_to_idx &&
@@ -473,9 +451,8 @@ int Net<Dtype>::AppendBottom(const NetParameter& param, const int layer_id,
<< layer_param.name() << "', bottom index " << bottom_id << ")";
}
const int blob_id = (*blob_name_to_idx)[blob_name];
- if (Caffe::root_solver()) {
- LOG(INFO) << layer_names_[layer_id] << " <- " << blob_name;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << layer_names_[layer_id] << " <- " << blob_name;
bottom_vecs_[layer_id].push_back(blobs_[blob_id].get());
bottom_id_vecs_[layer_id].push_back(blob_id);
available_blobs->erase(blob_name);
@@ -672,10 +649,9 @@ void Net<Dtype>::InputDebugInfo(const int input_id) {
const Blob<Dtype>& blob = *net_input_blobs_[input_id];
const string& blob_name = blob_names_[net_input_blob_indices_[input_id]];
const Dtype data_abs_val_mean = blob.asum_data() / blob.count();
- if (Caffe::root_solver()) {
- LOG(INFO) << " [Forward] "
- << "Input " << blob_name << " data: " << data_abs_val_mean;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " [Forward] "
+ << "Input " << blob_name << " data: " << data_abs_val_mean;
}
template <typename Dtype>
@@ -684,12 +660,11 @@ void Net<Dtype>::ForwardDebugInfo(const int layer_id) {
const Blob<Dtype>& blob = *top_vecs_[layer_id][top_id];
const string& blob_name = blob_names_[top_id_vecs_[layer_id][top_id]];
const Dtype data_abs_val_mean = blob.asum_data() / blob.count();
- if (Caffe::root_solver()) {
- LOG(INFO) << " [Forward] "
- << "Layer " << layer_names_[layer_id]
- << ", top blob " << blob_name
- << " data: " << data_abs_val_mean;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " [Forward] "
+ << "Layer " << layer_names_[layer_id]
+ << ", top blob " << blob_name
+ << " data: " << data_abs_val_mean;
}
for (int param_id = 0; param_id < layers_[layer_id]->blobs().size();
++param_id) {
@@ -697,12 +672,11 @@ void Net<Dtype>::ForwardDebugInfo(const int layer_id) {
const int net_param_id = param_id_vecs_[layer_id][param_id];
const string& blob_name = param_display_names_[net_param_id];
const Dtype data_abs_val_mean = blob.asum_data() / blob.count();
- if (Caffe::root_solver()) {
- LOG(INFO) << " [Forward] "
- << "Layer " << layer_names_[layer_id]
- << ", param blob " << blob_name
- << " data: " << data_abs_val_mean;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " [Forward] "
+ << "Layer " << layer_names_[layer_id]
+ << ", param blob " << blob_name
+ << " data: " << data_abs_val_mean;
}
}
@@ -714,24 +688,22 @@ void Net<Dtype>::BackwardDebugInfo(const int layer_id) {
const Blob<Dtype>& blob = *bottom_vec[bottom_id];
const string& blob_name = blob_names_[bottom_id_vecs_[layer_id][bottom_id]];
const Dtype diff_abs_val_mean = blob.asum_diff() / blob.count();
- if (Caffe::root_solver()) {
- LOG(INFO) << " [Backward] "
- << "Layer " << layer_names_[layer_id]
- << ", bottom blob " << blob_name
- << " diff: " << diff_abs_val_mean;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " [Backward] "
+ << "Layer " << layer_names_[layer_id]
+ << ", bottom blob " << blob_name
+ << " diff: " << diff_abs_val_mean;
}
for (int param_id = 0; param_id < layers_[layer_id]->blobs().size();
++param_id) {
if (!layers_[layer_id]->param_propagate_down(param_id)) { continue; }
const Blob<Dtype>& blob = *layers_[layer_id]->blobs()[param_id];
const Dtype diff_abs_val_mean = blob.asum_diff() / blob.count();
- if (Caffe::root_solver()) {
- LOG(INFO) << " [Backward] "
- << "Layer " << layer_names_[layer_id]
- << ", param blob " << param_id
- << " diff: " << diff_abs_val_mean;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " [Backward] "
+ << "Layer " << layer_names_[layer_id]
+ << ", param blob " << param_id
+ << " diff: " << diff_abs_val_mean;
}
}
@@ -744,22 +716,20 @@ void Net<Dtype>::UpdateDebugInfo(const int param_id) {
const Dtype diff_abs_val_mean = blob.asum_diff() / blob.count();
if (param_owner < 0) {
const Dtype data_abs_val_mean = blob.asum_data() / blob.count();
- if (Caffe::root_solver()) {
- LOG(INFO) << " [Update] Layer " << layer_name
- << ", param " << param_display_name
- << " data: " << data_abs_val_mean
- << "; diff: " << diff_abs_val_mean;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " [Update] Layer " << layer_name
+ << ", param " << param_display_name
+ << " data: " << data_abs_val_mean
+ << "; diff: " << diff_abs_val_mean;
} else {
const string& owner_layer_name =
layer_names_[param_layer_indices_[param_owner].first];
- if (Caffe::root_solver()) {
- LOG(INFO) << " [Update] Layer " << layer_name
- << ", param blob " << param_display_name
- << " (owned by layer " << owner_layer_name << ", " << "param "
- << param_display_names_[param_owners_[param_id]] << ")"
- << " diff: " << diff_abs_val_mean;
- }
+ LOG_IF(INFO, Caffe::root_solver())
+ << " [Update] Layer " << layer_name
+ << ", param blob " << param_display_name
+ << " (owned by layer " << owner_layer_name << ", " << "param "
+ << param_display_names_[param_owners_[param_id]] << ")"
+ << " diff: " << diff_abs_val_mean;
}
}
@@ -775,7 +745,7 @@ void Net<Dtype>::ShareTrainedLayersWith(const Net* other) {
++target_layer_id;
}
if (target_layer_id == layer_names_.size()) {
- DLOG(INFO) << "Ignoring source layer " << source_layer_name;
+ LOG(INFO) << "Ignoring source layer " << source_layer_name;
continue;
}
DLOG(INFO) << "Copying source layer " << source_layer_name;
@@ -843,7 +813,7 @@ void Net<Dtype>::CopyTrainedLayersFrom(const NetParameter& param) {
++target_layer_id;
}
if (target_layer_id == layer_names_.size()) {
- DLOG(INFO) << "Ignoring source layer " << source_layer_name;
+ LOG(INFO) << "Ignoring source layer " << source_layer_name;
continue;
}
DLOG(INFO) << "Copying source layer " << source_layer_name;
@@ -898,7 +868,7 @@ void Net<Dtype>::CopyTrainedLayersFromHDF5(const string trained_filename) {
for (int i = 0; i < num_layers; ++i) {
string source_layer_name = hdf5_get_name_by_idx(data_hid, i);
if (!layer_names_index_.count(source_layer_name)) {
- DLOG(INFO) << "Ignoring source layer " << source_layer_name;
+ LOG(INFO) << "Ignoring source layer " << source_layer_name;
continue;
}
int target_layer_id = layer_names_index_[source_layer_name];
diff --git a/src/caffe/parallel.cpp b/src/caffe/parallel.cpp
index a6d154e..62f5d73 100644
--- a/src/caffe/parallel.cpp
+++ b/src/caffe/parallel.cpp
@@ -3,11 +3,7 @@
#endif
#include <glog/logging.h>
#include <stdio.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <cstdlib>
#include <sstream>
#include <string>
#include <vector>
diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto
index f52c941..787369f 100644
--- a/src/caffe/proto/caffe.proto
+++ b/src/caffe/proto/caffe.proto
@@ -98,7 +98,7 @@ message NetParameter {
// NOTE
// Update the next available ID when you add a new SolverParameter field.
//
-// SolverParameter next available ID: 40 (last added: momentum2)
+// SolverParameter next available ID: 41 (last added: type)
message SolverParameter {
//////////////////////////////////////////////////////////////////////////////
// Specifying the train and test networks
@@ -209,16 +209,9 @@ message SolverParameter {
// (and by default) initialize using a seed derived from the system clock.
optional int64 random_seed = 20 [default = -1];
- // Solver type
- enum SolverType {
- SGD = 0;
- NESTEROV = 1;
- ADAGRAD = 2;
- RMSPROP = 3;
- ADADELTA = 4;
- ADAM = 5;
- }
- optional SolverType solver_type = 30 [default = SGD];
+ // type of the solver
+ optional string type = 40 [default = "SGD"];
+
// numerical stability for RMSProp, AdaGrad and AdaDelta and Adam
optional float delta = 31 [default = 1e-8];
// parameters for the Adam solver
@@ -234,6 +227,18 @@ message SolverParameter {
// If false, don't save a snapshot after training finishes.
optional bool snapshot_after_train = 28 [default = true];
+
+ // DEPRECATED: old solver enum types, use string instead
+ enum SolverType {
+ SGD = 0;
+ NESTEROV = 1;
+ ADAGRAD = 2;
+ RMSPROP = 3;
+ ADADELTA = 4;
+ ADAM = 5;
+ }
+ // DEPRECATED: use type instead of solver_type
+ optional SolverType solver_type = 30 [default = SGD];
}
// A message that stores the solver snapshots
@@ -301,7 +306,7 @@ message ParamSpec {
// NOTE
// Update the next available ID when you add a new LayerParameter field.
//
-// LayerParameter next available layer-specific ID: 139 (last added: tile_param)
+// LayerParameter next available layer-specific ID: 140 (last added: batch_norm_param)
message LayerParameter {
optional string name = 1; // the layer name
optional string type = 2; // the layer type
@@ -350,6 +355,7 @@ message LayerParameter {
// The default for the engine is set by the ENGINE switch at compile-time.
optional AccuracyParameter accuracy_param = 102;
optional ArgMaxParameter argmax_param = 103;
+ optional BatchNormParameter batch_norm_param = 139;
optional ConcatParameter concat_param = 104;
optional ContrastiveLossParameter contrastive_loss_param = 105;
optional ConvolutionParameter convolution_param = 106;
@@ -414,9 +420,27 @@ message TransformationParameter {
message LossParameter {
// If specified, ignore instances with the given label.
optional int32 ignore_label = 1;
- // If true, normalize each batch across all instances (including spatial
- // dimesions, but not ignored instances); else, divide by batch size only.
- optional bool normalize = 2 [default = true];
+ // How to normalize the loss for loss layers that aggregate across batches,
+ // spatial dimensions, or other dimensions. Currently only implemented in
+ // SoftmaxWithLoss layer.
+ enum NormalizationMode {
+ // Divide by the number of examples in the batch times spatial dimensions.
+ // Outputs that receive the ignore label will NOT be ignored in computing
+ // the normalization factor.
+ FULL = 0;
+ // Divide by the total number of output locations that do not take the
+ // ignore_label. If ignore_label is not set, this behaves like FULL.
+ VALID = 1;
+ // Divide by the batch size.
+ BATCH_SIZE = 2;
+ // Do not normalize the loss.
+ NONE = 3;
+ }
+ optional NormalizationMode normalization = 3 [default = VALID];
+ // Deprecated. Ignored if normalization is specified. If normalization
+ // is not specified, then setting this to false will be equivalent to
+ // normalization = BATCH_SIZE to be consistent with previous behavior.
+ optional bool normalize = 2;
}
// Messages that store parameters used by individual layer types follow, in
@@ -443,6 +467,11 @@ message ArgMaxParameter {
// If true produce pairs (argmax, maxval)
optional bool out_max_val = 1 [default = false];
optional uint32 top_k = 2 [default = 1];
+ // The axis along which to maximise -- may be negative to index from the
+ // end (e.g., -1 for the last axis).
+ // By default ArgMaxLayer maximizes over the flattened trailing dimensions
+ // for each index of the first / num dimension.
+ optional int32 axis = 3;
}
message ConcatParameter {
@@ -456,6 +485,18 @@ message ConcatParameter {
optional uint32 concat_dim = 1 [default = 1];
}
+message BatchNormParameter {
+ // If false, accumulate global mean/variance values via a moving average. If
+ // true, use those accumulated values instead of computing mean/variance
+ // across the batch.
+ optional bool use_global_stats = 1;
+ // How much does the moving average decay each iteration?
+ optional float moving_average_fraction = 2 [default = .999];
+ // Small value to add to the variance estimate so that we don't divide by
+ // zero.
+ optional float eps = 3 [default = 1e-5];
+}
+
message ContrastiveLossParameter {
// margin for dissimilar pair
optional float margin = 1 [default = 1.0];
@@ -721,6 +762,12 @@ message LRNParameter {
}
optional NormRegion norm_region = 4 [default = ACROSS_CHANNELS];
optional float k = 5 [default = 1.];
+ enum Engine {
+ DEFAULT = 0;
+ CAFFE = 1;
+ CUDNN = 2;
+ }
+ optional Engine engine = 6 [default = DEFAULT];
}
message MemoryDataParameter {
diff --git a/src/caffe/solver.cpp b/src/caffe/solver.cpp
index 12c13dd..95d7506 100644
--- a/src/caffe/solver.cpp
+++ b/src/caffe/solver.cpp
@@ -1,18 +1,12 @@
#include <cstdio>
-#include <algorithm>
#include <string>
#include <vector>
-#include "hdf5.h"
-#include "hdf5_hl.h"
-
-#include "caffe/net.hpp"
-#include "caffe/proto/caffe.pb.h"
#include "caffe/solver.hpp"
+#include "caffe/util/format.hpp"
#include "caffe/util/hdf5.hpp"
#include "caffe/util/io.hpp"
-#include "caffe/util/math_functions.hpp"
#include "caffe/util/upgrade_proto.hpp"
namespace caffe {
@@ -43,7 +37,7 @@ Solver<Dtype>::Solver(const string& param_file, const Solver* root_solver)
: net_(), callbacks_(), root_solver_(root_solver),
requested_early_exit_(false) {
SolverParameter param;
- ReadProtoFromTextFileOrDie(param_file, ¶m);
+ ReadSolverParamsFromTextFileOrDie(param_file, ¶m);
Init(param);
}
@@ -455,11 +449,8 @@ void Solver<Dtype>::CheckSnapshotWritePermissions() {
template <typename Dtype>
string Solver<Dtype>::SnapshotFilename(const string extension) {
- string filename(param_.snapshot_prefix());
- const int kBufferSize = 20;
- char iter_str_buffer[kBufferSize];
- snprintf(iter_str_buffer, kBufferSize, "_iter_%d", iter_);
- return filename + iter_str_buffer + extension;
+ return param_.snapshot_prefix() + "_iter_" + caffe::format_int(iter_)
+ + extension;
}
template <typename Dtype>
@@ -492,810 +483,6 @@ void Solver<Dtype>::Restore(const char* state_file) {
}
}
-// Return the current learning rate. The currently implemented learning rate
-// policies are as follows:
-// - fixed: always return base_lr.
-// - step: return base_lr * gamma ^ (floor(iter / step))
-// - exp: return base_lr * gamma ^ iter
-// - inv: return base_lr * (1 + gamma * iter) ^ (- power)
-// - multistep: similar to step but it allows non uniform steps defined by
-// stepvalue
-// - poly: the effective learning rate follows a polynomial decay, to be
-// zero by the max_iter. return base_lr (1 - iter/max_iter) ^ (power)
-// - sigmoid: the effective learning rate follows a sigmod decay
-// return base_lr ( 1/(1 + exp(-gamma * (iter - stepsize))))
-//
-// where base_lr, max_iter, gamma, step, stepvalue and power are defined
-// in the solver parameter protocol buffer, and iter is the current iteration.
-template <typename Dtype>
-Dtype SGDSolver<Dtype>::GetLearningRate() {
- Dtype rate;
- const string& lr_policy = this->param_.lr_policy();
- if (lr_policy == "fixed") {
- rate = this->param_.base_lr();
- } else if (lr_policy == "step") {
- this->current_step_ = this->iter_ / this->param_.stepsize();
- rate = this->param_.base_lr() *
- pow(this->param_.gamma(), this->current_step_);
- } else if (lr_policy == "exp") {
- rate = this->param_.base_lr() * pow(this->param_.gamma(), this->iter_);
- } else if (lr_policy == "inv") {
- rate = this->param_.base_lr() *
- pow(Dtype(1) + this->param_.gamma() * this->iter_,
- - this->param_.power());
- } else if (lr_policy == "multistep") {
- if (this->current_step_ < this->param_.stepvalue_size() &&
- this->iter_ >= this->param_.stepvalue(this->current_step_)) {
- this->current_step_++;
- LOG(INFO) << "MultiStep Status: Iteration " <<
- this->iter_ << ", step = " << this->current_step_;
- }
- rate = this->param_.base_lr() *
- pow(this->param_.gamma(), this->current_step_);
- } else if (lr_policy == "poly") {
- rate = this->param_.base_lr() * pow(Dtype(1.) -
- (Dtype(this->iter_) / Dtype(this->param_.max_iter())),
- this->param_.power());
- } else if (lr_policy == "sigmoid") {
- rate = this->param_.base_lr() * (Dtype(1.) /
- (Dtype(1.) + exp(-this->param_.gamma() * (Dtype(this->iter_) -
- Dtype(this->param_.stepsize())))));
- } else {
- LOG(FATAL) << "Unknown learning rate policy: " << lr_policy;
- }
- return rate;
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::PreSolve() {
- // Initialize the history
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- history_.clear();
- update_.clear();
- temp_.clear();
- for (int i = 0; i < net_params.size(); ++i) {
- const vector<int>& shape = net_params[i]->shape();
- history_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
- update_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
- temp_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
- }
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::ClipGradients() {
- const Dtype clip_gradients = this->param_.clip_gradients();
- if (clip_gradients < 0) { return; }
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- Dtype sumsq_diff = 0;
- for (int i = 0; i < net_params.size(); ++i) {
- sumsq_diff += net_params[i]->sumsq_diff();
- }
- const Dtype l2norm_diff = std::sqrt(sumsq_diff);
- if (l2norm_diff > clip_gradients) {
- Dtype scale_factor = clip_gradients / l2norm_diff;
- LOG(INFO) << "Gradient clipping: scaling down gradients (L2 norm "
- << l2norm_diff << " > " << clip_gradients << ") "
- << "by scale factor " << scale_factor;
- for (int i = 0; i < net_params.size(); ++i) {
- net_params[i]->scale_diff(scale_factor);
- }
- }
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::ApplyUpdate() {
- CHECK(Caffe::root_solver());
- Dtype rate = GetLearningRate();
- if (this->param_.display() && this->iter_ % this->param_.display() == 0) {
- LOG(INFO) << "Iteration " << this->iter_ << ", lr = " << rate;
- }
- ClipGradients();
- for (int param_id = 0; param_id < this->net_->learnable_params().size();
- ++param_id) {
- Normalize(param_id);
- Regularize(param_id);
- ComputeUpdateValue(param_id, rate);
- }
- this->net_->Update();
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::Normalize(int param_id) {
- if (this->param_.iter_size() == 1) { return; }
- // Scale gradient to counterbalance accumulation.
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const Dtype accum_normalization = Dtype(1.) / this->param_.iter_size();
- switch (Caffe::mode()) {
- case Caffe::CPU: {
- caffe_scal(net_params[param_id]->count(), accum_normalization,
- net_params[param_id]->mutable_cpu_diff());
- break;
- }
- case Caffe::GPU: {
-#ifndef CPU_ONLY
- caffe_gpu_scal(net_params[param_id]->count(), accum_normalization,
- net_params[param_id]->mutable_gpu_diff());
-#else
- NO_GPU;
-#endif
- break;
- }
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::Regularize(int param_id) {
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const vector<float>& net_params_weight_decay =
- this->net_->params_weight_decay();
- Dtype weight_decay = this->param_.weight_decay();
- string regularization_type = this->param_.regularization_type();
- Dtype local_decay = weight_decay * net_params_weight_decay[param_id];
- switch (Caffe::mode()) {
- case Caffe::CPU: {
- if (local_decay) {
- if (regularization_type == "L2") {
- // add weight decay
- caffe_axpy(net_params[param_id]->count(),
- local_decay,
- net_params[param_id]->cpu_data(),
- net_params[param_id]->mutable_cpu_diff());
- } else if (regularization_type == "L1") {
- caffe_cpu_sign(net_params[param_id]->count(),
- net_params[param_id]->cpu_data(),
- temp_[param_id]->mutable_cpu_data());
- caffe_axpy(net_params[param_id]->count(),
- local_decay,
- temp_[param_id]->cpu_data(),
- net_params[param_id]->mutable_cpu_diff());
- } else {
- LOG(FATAL) << "Unknown regularization type: " << regularization_type;
- }
- }
- break;
- }
- case Caffe::GPU: {
-#ifndef CPU_ONLY
- if (local_decay) {
- if (regularization_type == "L2") {
- // add weight decay
- caffe_gpu_axpy(net_params[param_id]->count(),
- local_decay,
- net_params[param_id]->gpu_data(),
- net_params[param_id]->mutable_gpu_diff());
- } else if (regularization_type == "L1") {
- caffe_gpu_sign(net_params[param_id]->count(),
- net_params[param_id]->gpu_data(),
- temp_[param_id]->mutable_gpu_data());
- caffe_gpu_axpy(net_params[param_id]->count(),
- local_decay,
- temp_[param_id]->gpu_data(),
- net_params[param_id]->mutable_gpu_diff());
- } else {
- LOG(FATAL) << "Unknown regularization type: " << regularization_type;
- }
- }
-#else
- NO_GPU;
-#endif
- break;
- }
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const vector<float>& net_params_lr = this->net_->params_lr();
- Dtype momentum = this->param_.momentum();
- Dtype local_rate = rate * net_params_lr[param_id];
- // Compute the update to history, then copy it to the parameter diff.
- switch (Caffe::mode()) {
- case Caffe::CPU: {
- caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
- net_params[param_id]->cpu_diff(), momentum,
- history_[param_id]->mutable_cpu_data());
- caffe_copy(net_params[param_id]->count(),
- history_[param_id]->cpu_data(),
- net_params[param_id]->mutable_cpu_diff());
- break;
- }
- case Caffe::GPU: {
-#ifndef CPU_ONLY
- caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
- net_params[param_id]->gpu_diff(), momentum,
- history_[param_id]->mutable_gpu_data());
- caffe_copy(net_params[param_id]->count(),
- history_[param_id]->gpu_data(),
- net_params[param_id]->mutable_gpu_diff());
-#else
- NO_GPU;
-#endif
- break;
- }
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::SnapshotSolverState(const string& model_filename) {
- switch (this->param_.snapshot_format()) {
- case caffe::SolverParameter_SnapshotFormat_BINARYPROTO:
- SnapshotSolverStateToBinaryProto(model_filename);
- break;
- case caffe::SolverParameter_SnapshotFormat_HDF5:
- SnapshotSolverStateToHDF5(model_filename);
- break;
- default:
- LOG(FATAL) << "Unsupported snapshot format.";
- }
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::SnapshotSolverStateToBinaryProto(
- const string& model_filename) {
- SolverState state;
- state.set_iter(this->iter_);
- state.set_learned_net(model_filename);
- state.set_current_step(this->current_step_);
- state.clear_history();
- for (int i = 0; i < history_.size(); ++i) {
- // Add history
- BlobProto* history_blob = state.add_history();
- history_[i]->ToProto(history_blob);
- }
- string snapshot_filename = Solver<Dtype>::SnapshotFilename(".solverstate");
- LOG(INFO)
- << "Snapshotting solver state to binary proto file " << snapshot_filename;
- WriteProtoToBinaryFile(state, snapshot_filename.c_str());
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::SnapshotSolverStateToHDF5(
- const string& model_filename) {
- string snapshot_filename =
- Solver<Dtype>::SnapshotFilename(".solverstate.h5");
- LOG(INFO) << "Snapshotting solver state to HDF5 file " << snapshot_filename;
- hid_t file_hid = H5Fcreate(snapshot_filename.c_str(), H5F_ACC_TRUNC,
- H5P_DEFAULT, H5P_DEFAULT);
- CHECK_GE(file_hid, 0)
- << "Couldn't open " << snapshot_filename << " to save solver state.";
- hdf5_save_int(file_hid, "iter", this->iter_);
- hdf5_save_string(file_hid, "learned_net", model_filename);
- hdf5_save_int(file_hid, "current_step", this->current_step_);
- hid_t history_hid = H5Gcreate2(file_hid, "history", H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT);
- CHECK_GE(history_hid, 0)
- << "Error saving solver state to " << snapshot_filename << ".";
- for (int i = 0; i < history_.size(); ++i) {
- ostringstream oss;
- oss << i;
- hdf5_save_nd_dataset<Dtype>(history_hid, oss.str(), *history_[i]);
- }
- H5Gclose(history_hid);
- H5Fclose(file_hid);
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::RestoreSolverStateFromBinaryProto(
- const string& state_file) {
- SolverState state;
- ReadProtoFromBinaryFile(state_file, &state);
- this->iter_ = state.iter();
- if (state.has_learned_net()) {
- NetParameter net_param;
- ReadNetParamsFromBinaryFileOrDie(state.learned_net().c_str(), &net_param);
- this->net_->CopyTrainedLayersFrom(net_param);
- }
- this->current_step_ = state.current_step();
- CHECK_EQ(state.history_size(), history_.size())
- << "Incorrect length of history blobs.";
- LOG(INFO) << "SGDSolver: restoring history";
- for (int i = 0; i < history_.size(); ++i) {
- history_[i]->FromProto(state.history(i));
- }
-}
-
-template <typename Dtype>
-void SGDSolver<Dtype>::RestoreSolverStateFromHDF5(const string& state_file) {
- hid_t file_hid = H5Fopen(state_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
- CHECK_GE(file_hid, 0) << "Couldn't open solver state file " << state_file;
- this->iter_ = hdf5_load_int(file_hid, "iter");
- if (H5LTfind_dataset(file_hid, "learned_net")) {
- string learned_net = hdf5_load_string(file_hid, "learned_net");
- this->net_->CopyTrainedLayersFrom(learned_net);
- }
- this->current_step_ = hdf5_load_int(file_hid, "current_step");
- hid_t history_hid = H5Gopen2(file_hid, "history", H5P_DEFAULT);
- CHECK_GE(history_hid, 0) << "Error reading history from " << state_file;
- int state_history_size = hdf5_get_num_links(history_hid);
- CHECK_EQ(state_history_size, history_.size())
- << "Incorrect length of history blobs.";
- for (int i = 0; i < history_.size(); ++i) {
- ostringstream oss;
- oss << i;
- hdf5_load_nd_dataset<Dtype>(history_hid, oss.str().c_str(), 0,
- kMaxBlobAxes, history_[i].get());
- }
- H5Gclose(history_hid);
- H5Fclose(file_hid);
-}
-
-template <typename Dtype>
-void NesterovSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
- CHECK(Caffe::root_solver());
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const vector<float>& net_params_lr = this->net_->params_lr();
- Dtype momentum = this->param_.momentum();
- Dtype local_rate = rate * net_params_lr[param_id];
- switch (Caffe::mode()) {
- case Caffe::CPU: {
- // save history momentum for stepping back
- caffe_copy(net_params[param_id]->count(),
- this->history_[param_id]->cpu_data(),
- this->update_[param_id]->mutable_cpu_data());
-
- // update history
- caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
- net_params[param_id]->cpu_diff(), momentum,
- this->history_[param_id]->mutable_cpu_data());
-
- // compute update: step back then over step
- caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum,
- this->history_[param_id]->cpu_data(), -momentum,
- this->update_[param_id]->mutable_cpu_data());
-
- // copy
- caffe_copy(net_params[param_id]->count(),
- this->update_[param_id]->cpu_data(),
- net_params[param_id]->mutable_cpu_diff());
- break;
- }
- case Caffe::GPU: {
-#ifndef CPU_ONLY
- // save history momentum for stepping back
- caffe_copy(net_params[param_id]->count(),
- this->history_[param_id]->gpu_data(),
- this->update_[param_id]->mutable_gpu_data());
-
- // update history
- caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
- net_params[param_id]->gpu_diff(), momentum,
- this->history_[param_id]->mutable_gpu_data());
-
- // compute update: step back then over step
- caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum,
- this->history_[param_id]->gpu_data(), -momentum,
- this->update_[param_id]->mutable_gpu_data());
-
- // copy
- caffe_copy(net_params[param_id]->count(),
- this->update_[param_id]->gpu_data(),
- net_params[param_id]->mutable_gpu_diff());
-#else
- NO_GPU;
-#endif
- break;
- }
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
-template <typename Dtype>
-void AdaGradSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
- CHECK(Caffe::root_solver());
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const vector<float>& net_params_lr = this->net_->params_lr();
- Dtype delta = this->param_.delta();
- Dtype local_rate = rate * net_params_lr[param_id];
- switch (Caffe::mode()) {
- case Caffe::CPU: {
- // compute square of gradient in update
- caffe_powx(net_params[param_id]->count(),
- net_params[param_id]->cpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_cpu_data());
-
- // update history
- caffe_add(net_params[param_id]->count(),
- this->update_[param_id]->cpu_data(),
- this->history_[param_id]->cpu_data(),
- this->history_[param_id]->mutable_cpu_data());
-
- // prepare update
- caffe_powx(net_params[param_id]->count(),
- this->history_[param_id]->cpu_data(), Dtype(0.5),
- this->update_[param_id]->mutable_cpu_data());
-
- caffe_add_scalar(net_params[param_id]->count(),
- delta, this->update_[param_id]->mutable_cpu_data());
-
- caffe_div(net_params[param_id]->count(),
- net_params[param_id]->cpu_diff(),
- this->update_[param_id]->cpu_data(),
- this->update_[param_id]->mutable_cpu_data());
-
- // scale and copy
- caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
- this->update_[param_id]->cpu_data(), Dtype(0),
- net_params[param_id]->mutable_cpu_diff());
- break;
- }
- case Caffe::GPU: {
-#ifndef CPU_ONLY
- // compute square of gradient in update
- caffe_gpu_powx(net_params[param_id]->count(),
- net_params[param_id]->gpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_gpu_data());
-
- // update history
- caffe_gpu_add(net_params[param_id]->count(),
- this->update_[param_id]->gpu_data(),
- this->history_[param_id]->gpu_data(),
- this->history_[param_id]->mutable_gpu_data());
-
- // prepare update
- caffe_gpu_powx(net_params[param_id]->count(),
- this->history_[param_id]->gpu_data(), Dtype(0.5),
- this->update_[param_id]->mutable_gpu_data());
-
- caffe_gpu_add_scalar(net_params[param_id]->count(),
- delta, this->update_[param_id]->mutable_gpu_data());
-
- caffe_gpu_div(net_params[param_id]->count(),
- net_params[param_id]->gpu_diff(),
- this->update_[param_id]->gpu_data(),
- this->update_[param_id]->mutable_gpu_data());
-
- // scale and copy
- caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
- this->update_[param_id]->gpu_data(), Dtype(0),
- net_params[param_id]->mutable_gpu_diff());
-#else
- NO_GPU;
-#endif
- break;
- }
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
-template <typename Dtype>
-void RMSPropSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const vector<float>& net_params_lr = this->net_->params_lr();
-
- // get the learning rate
- Dtype delta = this->param_.delta();
- Dtype rms_decay = this->param_.rms_decay();
- Dtype local_rate = rate * net_params_lr[param_id];
-
- switch (Caffe::mode()) {
- case Caffe::CPU:
- // compute square of gradient in update
- caffe_powx(net_params[param_id]->count(),
- net_params[param_id]->cpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_cpu_data());
-
- // update history
- caffe_cpu_axpby(net_params[param_id] -> count(),
- Dtype(1-rms_decay), this->update_[param_id]->cpu_data(),
- rms_decay, this->history_[param_id]-> mutable_cpu_data());
-
- // prepare update
- caffe_powx(net_params[param_id]->count(),
- this->history_[param_id]->cpu_data(), Dtype(0.5),
- this->update_[param_id]->mutable_cpu_data());
-
- caffe_add_scalar(net_params[param_id]->count(),
- delta, this->update_[param_id]->mutable_cpu_data());
-
- caffe_div(net_params[param_id]->count(),
- net_params[param_id]->cpu_diff(), this->update_[param_id]->cpu_data(),
- this->update_[param_id]->mutable_cpu_data());
-
- // scale and copy
- caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
- this->update_[param_id]->cpu_data(), Dtype(0),
- net_params[param_id]->mutable_cpu_diff());
- break;
- case Caffe::GPU:
-#ifndef CPU_ONLY
- // compute square of gradient in update
- caffe_gpu_powx(net_params[param_id]->count(),
- net_params[param_id]->gpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_gpu_data());
-
- // update history
- caffe_gpu_axpby(net_params[param_id] -> count(),
- Dtype(1-rms_decay), this->update_[param_id]->gpu_data(),
- rms_decay, this->history_[param_id]-> mutable_gpu_data());
-
- // prepare update
- caffe_gpu_powx(net_params[param_id]->count(),
- this->history_[param_id]->gpu_data(), Dtype(0.5),
- this->update_[param_id]->mutable_gpu_data());
-
- caffe_gpu_add_scalar(net_params[param_id]->count(),
- delta, this->update_[param_id]->mutable_gpu_data());
-
- caffe_gpu_div(net_params[param_id]->count(),
- net_params[param_id]->gpu_diff(), this->update_[param_id]->gpu_data(),
- this->update_[param_id]->mutable_gpu_data());
-
- caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
- this->update_[param_id]->gpu_data(), Dtype(0),
- net_params[param_id]->mutable_gpu_diff());
-#else
- NO_GPU;
-#endif
- break;
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
-template <typename Dtype>
-void AdaDeltaSolver<Dtype>::AdaDeltaPreSolve() {
- // Add the extra history entries for AdaDelta after those from
- // SGDSolver::PreSolve
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- for (int i = 0; i < net_params.size(); ++i) {
- const vector<int>& shape = net_params[i]->shape();
- this->history_.push_back(
- shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
- }
-}
-
-template <typename Dtype>
-void AdaDeltaSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const vector<float>& net_params_lr = this->net_->params_lr();
- Dtype delta = this->param_.delta();
- Dtype momentum = this->param_.momentum();
- Dtype local_rate = rate * net_params_lr[param_id];
- size_t update_history_offset = net_params.size();
- switch (Caffe::mode()) {
- case Caffe::CPU: {
- // compute square of gradient in update
- caffe_powx(net_params[param_id]->count(),
- net_params[param_id]->cpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_cpu_data());
-
- // update history of gradients
- caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
- this->update_[param_id]->cpu_data(), momentum,
- this->history_[param_id]->mutable_cpu_data());
-
- // add delta to history to guard against dividing by zero later
- caffe_set(net_params[param_id]->count(), delta,
- this->temp_[param_id]->mutable_cpu_data());
-
- caffe_add(net_params[param_id]->count(),
- this->temp_[param_id]->cpu_data(),
- this->history_[update_history_offset + param_id]->cpu_data(),
- this->update_[param_id]->mutable_cpu_data());
-
- caffe_add(net_params[param_id]->count(),
- this->temp_[param_id]->cpu_data(),
- this->history_[param_id]->cpu_data(),
- this->temp_[param_id]->mutable_cpu_data());
-
- // divide history of updates by history of gradients
- caffe_div(net_params[param_id]->count(),
- this->update_[param_id]->cpu_data(),
- this->temp_[param_id]->cpu_data(),
- this->update_[param_id]->mutable_cpu_data());
-
- // jointly compute the RMS of both for update and gradient history
- caffe_powx(net_params[param_id]->count(),
- this->update_[param_id]->cpu_data(), Dtype(0.5),
- this->update_[param_id]->mutable_cpu_data());
-
- // compute the update
- caffe_mul(net_params[param_id]->count(),
- net_params[param_id]->cpu_diff(),
- this->update_[param_id]->cpu_data(),
- net_params[param_id]->mutable_cpu_diff());
-
- // compute square of update
- caffe_powx(net_params[param_id]->count(),
- net_params[param_id]->cpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_cpu_data());
-
- // update history of updates
- caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
- this->update_[param_id]->cpu_data(), momentum,
- this->history_[update_history_offset + param_id]->mutable_cpu_data());
-
- // apply learning rate
- caffe_cpu_scale(net_params[param_id]->count(), local_rate,
- net_params[param_id]->cpu_diff(),
- net_params[param_id]->mutable_cpu_diff());
- break;
- }
- case Caffe::GPU: {
-#ifndef CPU_ONLY
- // compute square of gradient in update
- caffe_gpu_powx(net_params[param_id]->count(),
- net_params[param_id]->gpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_gpu_data());
-
- // update history of gradients
- caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
- this->update_[param_id]->gpu_data(), momentum,
- this->history_[param_id]->mutable_gpu_data());
-
- // add delta to history to guard against dividing by zero later
- caffe_gpu_set(net_params[param_id]->count(), delta,
- this->temp_[param_id]->mutable_gpu_data());
-
- caffe_gpu_add(net_params[param_id]->count(),
- this->temp_[param_id]->gpu_data(),
- this->history_[update_history_offset + param_id]->gpu_data(),
- this->update_[param_id]->mutable_gpu_data());
-
- caffe_gpu_add(net_params[param_id]->count(),
- this->temp_[param_id]->gpu_data(),
- this->history_[param_id]->gpu_data(),
- this->temp_[param_id]->mutable_gpu_data());
-
- // divide history of updates by history of gradients
- caffe_gpu_div(net_params[param_id]->count(),
- this->update_[param_id]->gpu_data(),
- this->temp_[param_id]->gpu_data(),
- this->update_[param_id]->mutable_gpu_data());
-
- // jointly compute the RMS of both for update and gradient history
- caffe_gpu_powx(net_params[param_id]->count(),
- this->update_[param_id]->gpu_data(), Dtype(0.5),
- this->update_[param_id]->mutable_gpu_data());
-
- // compute the update and copy to net_diff
- caffe_gpu_mul(net_params[param_id]->count(),
- net_params[param_id]->gpu_diff(),
- this->update_[param_id]->gpu_data(),
- net_params[param_id]->mutable_gpu_diff());
-
- // compute square of update
- caffe_gpu_powx(net_params[param_id]->count(),
- net_params[param_id]->gpu_diff(), Dtype(2),
- this->update_[param_id]->mutable_gpu_data());
-
- // update history of updates
- caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
- this->update_[param_id]->gpu_data(), momentum,
- this->history_[update_history_offset + param_id]->mutable_gpu_data());
-
- // apply learning rate
- caffe_gpu_scale(net_params[param_id]->count(), local_rate,
- net_params[param_id]->gpu_diff(),
- net_params[param_id]->mutable_gpu_diff());
-#else
- NO_GPU;
-#endif
- break;
- }
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
-template <typename Dtype>
-void AdamSolver<Dtype>::AdamPreSolve() {
- // Add the extra history entries for Adam after those from
- // SGDSolver::PreSolve
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- for (int i = 0; i < net_params.size(); ++i) {
- const vector<int>& shape = net_params[i]->shape();
- this->history_.push_back(
- shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
- }
-}
-
-template <typename Dtype>
-void AdamSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
- const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
- const vector<float>& net_params_lr = this->net_->params_lr();
- Dtype local_rate = rate * net_params_lr[param_id];
- const Dtype beta1 = this->param_.momentum();
- const Dtype beta2 = this->param_.momentum2();
-
- // we create aliases for convenience
- size_t update_history_offset = net_params.size();
- Blob<Dtype>* val_m = this->history_[param_id].get();
- Blob<Dtype>* val_v = this->history_[param_id + update_history_offset].get();
- Blob<Dtype>* val_t = this->temp_[param_id].get();
-
- const int t = this->iter_ + 1;
- const Dtype correction = std::sqrt(Dtype(1) - pow(beta2, t)) /
- (Dtype(1.) - pow(beta1, t));
- const int N = net_params[param_id]->count();
- const Dtype eps_hat = this->param_.delta();
-
- switch (Caffe::mode()) {
- case Caffe::CPU: {
- // update m <- \beta_1 m_{t-1} + (1-\beta_1)g_t
- caffe_cpu_axpby(N, Dtype(1)-beta1,
- net_params[param_id]->cpu_diff(), beta1,
- val_m->mutable_cpu_data());
-
- // update v <- \beta_2 m_{t-1} + (1-\beta_2)g_t^2
- caffe_mul(N,
- net_params[param_id]->cpu_diff(),
- net_params[param_id]->cpu_diff(),
- val_t->mutable_cpu_data());
- caffe_cpu_axpby(N, Dtype(1)-beta2,
- val_t->cpu_data(), beta2,
- val_v->mutable_cpu_data());
-
- // set update
- caffe_powx(N,
- val_v->cpu_data(), Dtype(0.5),
- val_t->mutable_cpu_data());
- caffe_add_scalar(N, eps_hat, val_t->mutable_cpu_data());
- caffe_div(N,
- val_m->cpu_data(),
- val_t->cpu_data(),
- val_t->mutable_cpu_data());
-
- caffe_cpu_scale(N, local_rate*correction,
- val_t->cpu_data(),
- net_params[param_id]->mutable_cpu_diff());
- break;
- }
- case Caffe::GPU: {
-#ifndef CPU_ONLY
- // update m <- \beta_1 m_{t-1} + (1-\beta_1)g_t
- caffe_gpu_axpby(N, Dtype(1)-beta1,
- net_params[param_id]->gpu_diff(), beta1,
- val_m->mutable_gpu_data());
-
- // update v <- \beta_2 m_{t-1} + (1-\beta_2)g_t^2
- caffe_gpu_mul(N,
- net_params[param_id]->gpu_diff(),
- net_params[param_id]->gpu_diff(),
- val_t->mutable_gpu_data());
- caffe_gpu_axpby(N, Dtype(1)-beta2,
- val_t->gpu_data(), beta2,
- val_v->mutable_gpu_data());
-
- // set update
- caffe_gpu_powx(N,
- val_v->gpu_data(), Dtype(0.5),
- val_t->mutable_gpu_data());
- caffe_gpu_add_scalar(N, eps_hat,
- val_t->mutable_gpu_data());
- caffe_gpu_div(N,
- val_m->gpu_data(),
- val_t->gpu_data(),
- val_t->mutable_gpu_data());
-
- caffe_gpu_scale(N, local_rate*correction,
- val_t->gpu_data(),
- net_params[param_id]->mutable_gpu_diff());
-#else
- NO_GPU;
-#endif
- break;
- }
- default:
- LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
- }
-}
-
INSTANTIATE_CLASS(Solver);
-INSTANTIATE_CLASS(SGDSolver);
-INSTANTIATE_CLASS(NesterovSolver);
-INSTANTIATE_CLASS(AdaGradSolver);
-INSTANTIATE_CLASS(RMSPropSolver);
-INSTANTIATE_CLASS(AdaDeltaSolver);
-INSTANTIATE_CLASS(AdamSolver);
} // namespace caffe
diff --git a/src/caffe/solvers/adadelta_solver.cpp b/src/caffe/solvers/adadelta_solver.cpp
new file mode 100644
index 0000000..a37899e
--- /dev/null
+++ b/src/caffe/solvers/adadelta_solver.cpp
@@ -0,0 +1,156 @@
+#include <vector>
+
+#include "caffe/sgd_solvers.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void AdaDeltaSolver<Dtype>::AdaDeltaPreSolve() {
+ // Add the extra history entries for AdaDelta after those from
+ // SGDSolver::PreSolve
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ for (int i = 0; i < net_params.size(); ++i) {
+ const vector<int>& shape = net_params[i]->shape();
+ this->history_.push_back(
+ shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
+ }
+}
+
+template <typename Dtype>
+void AdaDeltaSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_lr = this->net_->params_lr();
+ Dtype delta = this->param_.delta();
+ Dtype momentum = this->param_.momentum();
+ Dtype local_rate = rate * net_params_lr[param_id];
+ size_t update_history_offset = net_params.size();
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ // compute square of gradient in update
+ caffe_powx(net_params[param_id]->count(),
+ net_params[param_id]->cpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // update history of gradients
+ caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
+ this->update_[param_id]->cpu_data(), momentum,
+ this->history_[param_id]->mutable_cpu_data());
+
+ // add delta to history to guard against dividing by zero later
+ caffe_set(net_params[param_id]->count(), delta,
+ this->temp_[param_id]->mutable_cpu_data());
+
+ caffe_add(net_params[param_id]->count(),
+ this->temp_[param_id]->cpu_data(),
+ this->history_[update_history_offset + param_id]->cpu_data(),
+ this->update_[param_id]->mutable_cpu_data());
+
+ caffe_add(net_params[param_id]->count(),
+ this->temp_[param_id]->cpu_data(),
+ this->history_[param_id]->cpu_data(),
+ this->temp_[param_id]->mutable_cpu_data());
+
+ // divide history of updates by history of gradients
+ caffe_div(net_params[param_id]->count(),
+ this->update_[param_id]->cpu_data(),
+ this->temp_[param_id]->cpu_data(),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // jointly compute the RMS of both for update and gradient history
+ caffe_powx(net_params[param_id]->count(),
+ this->update_[param_id]->cpu_data(), Dtype(0.5),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // compute the update
+ caffe_mul(net_params[param_id]->count(),
+ net_params[param_id]->cpu_diff(),
+ this->update_[param_id]->cpu_data(),
+ net_params[param_id]->mutable_cpu_diff());
+
+ // compute square of update
+ caffe_powx(net_params[param_id]->count(),
+ net_params[param_id]->cpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // update history of updates
+ caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
+ this->update_[param_id]->cpu_data(), momentum,
+ this->history_[update_history_offset + param_id]->mutable_cpu_data());
+
+ // apply learning rate
+ caffe_cpu_scale(net_params[param_id]->count(), local_rate,
+ net_params[param_id]->cpu_diff(),
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ // compute square of gradient in update
+ caffe_gpu_powx(net_params[param_id]->count(),
+ net_params[param_id]->gpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // update history of gradients
+ caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
+ this->update_[param_id]->gpu_data(), momentum,
+ this->history_[param_id]->mutable_gpu_data());
+
+ // add delta to history to guard against dividing by zero later
+ caffe_gpu_set(net_params[param_id]->count(), delta,
+ this->temp_[param_id]->mutable_gpu_data());
+
+ caffe_gpu_add(net_params[param_id]->count(),
+ this->temp_[param_id]->gpu_data(),
+ this->history_[update_history_offset + param_id]->gpu_data(),
+ this->update_[param_id]->mutable_gpu_data());
+
+ caffe_gpu_add(net_params[param_id]->count(),
+ this->temp_[param_id]->gpu_data(),
+ this->history_[param_id]->gpu_data(),
+ this->temp_[param_id]->mutable_gpu_data());
+
+ // divide history of updates by history of gradients
+ caffe_gpu_div(net_params[param_id]->count(),
+ this->update_[param_id]->gpu_data(),
+ this->temp_[param_id]->gpu_data(),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // jointly compute the RMS of both for update and gradient history
+ caffe_gpu_powx(net_params[param_id]->count(),
+ this->update_[param_id]->gpu_data(), Dtype(0.5),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // compute the update and copy to net_diff
+ caffe_gpu_mul(net_params[param_id]->count(),
+ net_params[param_id]->gpu_diff(),
+ this->update_[param_id]->gpu_data(),
+ net_params[param_id]->mutable_gpu_diff());
+
+ // compute square of update
+ caffe_gpu_powx(net_params[param_id]->count(),
+ net_params[param_id]->gpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // update history of updates
+ caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) - momentum,
+ this->update_[param_id]->gpu_data(), momentum,
+ this->history_[update_history_offset + param_id]->mutable_gpu_data());
+
+ // apply learning rate
+ caffe_gpu_scale(net_params[param_id]->count(), local_rate,
+ net_params[param_id]->gpu_diff(),
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+INSTANTIATE_CLASS(AdaDeltaSolver);
+REGISTER_SOLVER_CLASS(AdaDelta);
+
+} // namespace caffe
diff --git a/src/caffe/solvers/adagrad_solver.cpp b/src/caffe/solvers/adagrad_solver.cpp
new file mode 100644
index 0000000..5e40632
--- /dev/null
+++ b/src/caffe/solvers/adagrad_solver.cpp
@@ -0,0 +1,89 @@
+#include <vector>
+
+#include "caffe/sgd_solvers.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void AdaGradSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
+ CHECK(Caffe::root_solver());
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_lr = this->net_->params_lr();
+ Dtype delta = this->param_.delta();
+ Dtype local_rate = rate * net_params_lr[param_id];
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ // compute square of gradient in update
+ caffe_powx(net_params[param_id]->count(),
+ net_params[param_id]->cpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // update history
+ caffe_add(net_params[param_id]->count(),
+ this->update_[param_id]->cpu_data(),
+ this->history_[param_id]->cpu_data(),
+ this->history_[param_id]->mutable_cpu_data());
+
+ // prepare update
+ caffe_powx(net_params[param_id]->count(),
+ this->history_[param_id]->cpu_data(), Dtype(0.5),
+ this->update_[param_id]->mutable_cpu_data());
+
+ caffe_add_scalar(net_params[param_id]->count(),
+ delta, this->update_[param_id]->mutable_cpu_data());
+
+ caffe_div(net_params[param_id]->count(),
+ net_params[param_id]->cpu_diff(),
+ this->update_[param_id]->cpu_data(),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // scale and copy
+ caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
+ this->update_[param_id]->cpu_data(), Dtype(0),
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ // compute square of gradient in update
+ caffe_gpu_powx(net_params[param_id]->count(),
+ net_params[param_id]->gpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // update history
+ caffe_gpu_add(net_params[param_id]->count(),
+ this->update_[param_id]->gpu_data(),
+ this->history_[param_id]->gpu_data(),
+ this->history_[param_id]->mutable_gpu_data());
+
+ // prepare update
+ caffe_gpu_powx(net_params[param_id]->count(),
+ this->history_[param_id]->gpu_data(), Dtype(0.5),
+ this->update_[param_id]->mutable_gpu_data());
+
+ caffe_gpu_add_scalar(net_params[param_id]->count(),
+ delta, this->update_[param_id]->mutable_gpu_data());
+
+ caffe_gpu_div(net_params[param_id]->count(),
+ net_params[param_id]->gpu_diff(),
+ this->update_[param_id]->gpu_data(),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // scale and copy
+ caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
+ this->update_[param_id]->gpu_data(), Dtype(0),
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+INSTANTIATE_CLASS(AdaGradSolver);
+REGISTER_SOLVER_CLASS(AdaGrad);
+
+} // namespace caffe
diff --git a/src/caffe/solvers/adam_solver.cpp b/src/caffe/solvers/adam_solver.cpp
new file mode 100644
index 0000000..cb0fbfe
--- /dev/null
+++ b/src/caffe/solvers/adam_solver.cpp
@@ -0,0 +1,113 @@
+#include <vector>
+
+#include "caffe/sgd_solvers.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void AdamSolver<Dtype>::AdamPreSolve() {
+ // Add the extra history entries for Adam after those from
+ // SGDSolver::PreSolve
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ for (int i = 0; i < net_params.size(); ++i) {
+ const vector<int>& shape = net_params[i]->shape();
+ this->history_.push_back(
+ shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
+ }
+}
+
+template <typename Dtype>
+void AdamSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_lr = this->net_->params_lr();
+ Dtype local_rate = rate * net_params_lr[param_id];
+ const Dtype beta1 = this->param_.momentum();
+ const Dtype beta2 = this->param_.momentum2();
+
+ // we create aliases for convenience
+ size_t update_history_offset = net_params.size();
+ Blob<Dtype>* val_m = this->history_[param_id].get();
+ Blob<Dtype>* val_v = this->history_[param_id + update_history_offset].get();
+ Blob<Dtype>* val_t = this->temp_[param_id].get();
+
+ const int t = this->iter_ + 1;
+ const Dtype correction = std::sqrt(Dtype(1) - pow(beta2, t)) /
+ (Dtype(1.) - pow(beta1, t));
+ const int N = net_params[param_id]->count();
+ const Dtype eps_hat = this->param_.delta();
+
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ // update m <- \beta_1 m_{t-1} + (1-\beta_1)g_t
+ caffe_cpu_axpby(N, Dtype(1)-beta1,
+ net_params[param_id]->cpu_diff(), beta1,
+ val_m->mutable_cpu_data());
+
+ // update v <- \beta_2 m_{t-1} + (1-\beta_2)g_t^2
+ caffe_mul(N,
+ net_params[param_id]->cpu_diff(),
+ net_params[param_id]->cpu_diff(),
+ val_t->mutable_cpu_data());
+ caffe_cpu_axpby(N, Dtype(1)-beta2,
+ val_t->cpu_data(), beta2,
+ val_v->mutable_cpu_data());
+
+ // set update
+ caffe_powx(N,
+ val_v->cpu_data(), Dtype(0.5),
+ val_t->mutable_cpu_data());
+ caffe_add_scalar(N, eps_hat, val_t->mutable_cpu_data());
+ caffe_div(N,
+ val_m->cpu_data(),
+ val_t->cpu_data(),
+ val_t->mutable_cpu_data());
+
+ caffe_cpu_scale(N, local_rate*correction,
+ val_t->cpu_data(),
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ // update m <- \beta_1 m_{t-1} + (1-\beta_1)g_t
+ caffe_gpu_axpby(N, Dtype(1)-beta1,
+ net_params[param_id]->gpu_diff(), beta1,
+ val_m->mutable_gpu_data());
+
+ // update v <- \beta_2 m_{t-1} + (1-\beta_2)g_t^2
+ caffe_gpu_mul(N,
+ net_params[param_id]->gpu_diff(),
+ net_params[param_id]->gpu_diff(),
+ val_t->mutable_gpu_data());
+ caffe_gpu_axpby(N, Dtype(1)-beta2,
+ val_t->gpu_data(), beta2,
+ val_v->mutable_gpu_data());
+
+ // set update
+ caffe_gpu_powx(N,
+ val_v->gpu_data(), Dtype(0.5),
+ val_t->mutable_gpu_data());
+ caffe_gpu_add_scalar(N, eps_hat,
+ val_t->mutable_gpu_data());
+ caffe_gpu_div(N,
+ val_m->gpu_data(),
+ val_t->gpu_data(),
+ val_t->mutable_gpu_data());
+
+ caffe_gpu_scale(N, local_rate*correction,
+ val_t->gpu_data(),
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+INSTANTIATE_CLASS(AdamSolver);
+REGISTER_SOLVER_CLASS(Adam);
+
+} // namespace caffe
diff --git a/src/caffe/solvers/nesterov_solver.cpp b/src/caffe/solvers/nesterov_solver.cpp
new file mode 100644
index 0000000..34bf01e
--- /dev/null
+++ b/src/caffe/solvers/nesterov_solver.cpp
@@ -0,0 +1,71 @@
+#include <vector>
+
+#include "caffe/sgd_solvers.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void NesterovSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
+ CHECK(Caffe::root_solver());
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_lr = this->net_->params_lr();
+ Dtype momentum = this->param_.momentum();
+ Dtype local_rate = rate * net_params_lr[param_id];
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ // save history momentum for stepping back
+ caffe_copy(net_params[param_id]->count(),
+ this->history_[param_id]->cpu_data(),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // update history
+ caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
+ net_params[param_id]->cpu_diff(), momentum,
+ this->history_[param_id]->mutable_cpu_data());
+
+ // compute update: step back then over step
+ caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum,
+ this->history_[param_id]->cpu_data(), -momentum,
+ this->update_[param_id]->mutable_cpu_data());
+
+ // copy
+ caffe_copy(net_params[param_id]->count(),
+ this->update_[param_id]->cpu_data(),
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ // save history momentum for stepping back
+ caffe_copy(net_params[param_id]->count(),
+ this->history_[param_id]->gpu_data(),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // update history
+ caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
+ net_params[param_id]->gpu_diff(), momentum,
+ this->history_[param_id]->mutable_gpu_data());
+
+ // compute update: step back then over step
+ caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum,
+ this->history_[param_id]->gpu_data(), -momentum,
+ this->update_[param_id]->mutable_gpu_data());
+
+ // copy
+ caffe_copy(net_params[param_id]->count(),
+ this->update_[param_id]->gpu_data(),
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+INSTANTIATE_CLASS(NesterovSolver);
+REGISTER_SOLVER_CLASS(Nesterov);
+
+} // namespace caffe
diff --git a/src/caffe/solvers/rmsprop_solver.cpp b/src/caffe/solvers/rmsprop_solver.cpp
new file mode 100644
index 0000000..c624767
--- /dev/null
+++ b/src/caffe/solvers/rmsprop_solver.cpp
@@ -0,0 +1,85 @@
+#include <vector>
+
+#include "caffe/sgd_solvers.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void RMSPropSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_lr = this->net_->params_lr();
+
+ // get the learning rate
+ Dtype delta = this->param_.delta();
+ Dtype rms_decay = this->param_.rms_decay();
+ Dtype local_rate = rate * net_params_lr[param_id];
+
+ switch (Caffe::mode()) {
+ case Caffe::CPU:
+ // compute square of gradient in update
+ caffe_powx(net_params[param_id]->count(),
+ net_params[param_id]->cpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // update history
+ caffe_cpu_axpby(net_params[param_id] -> count(),
+ Dtype(1-rms_decay), this->update_[param_id]->cpu_data(),
+ rms_decay, this->history_[param_id]-> mutable_cpu_data());
+
+ // prepare update
+ caffe_powx(net_params[param_id]->count(),
+ this->history_[param_id]->cpu_data(), Dtype(0.5),
+ this->update_[param_id]->mutable_cpu_data());
+
+ caffe_add_scalar(net_params[param_id]->count(),
+ delta, this->update_[param_id]->mutable_cpu_data());
+
+ caffe_div(net_params[param_id]->count(),
+ net_params[param_id]->cpu_diff(), this->update_[param_id]->cpu_data(),
+ this->update_[param_id]->mutable_cpu_data());
+
+ // scale and copy
+ caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
+ this->update_[param_id]->cpu_data(), Dtype(0),
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ case Caffe::GPU:
+#ifndef CPU_ONLY
+ // compute square of gradient in update
+ caffe_gpu_powx(net_params[param_id]->count(),
+ net_params[param_id]->gpu_diff(), Dtype(2),
+ this->update_[param_id]->mutable_gpu_data());
+
+ // update history
+ caffe_gpu_axpby(net_params[param_id] -> count(),
+ Dtype(1-rms_decay), this->update_[param_id]->gpu_data(),
+ rms_decay, this->history_[param_id]-> mutable_gpu_data());
+
+ // prepare update
+ caffe_gpu_powx(net_params[param_id]->count(),
+ this->history_[param_id]->gpu_data(), Dtype(0.5),
+ this->update_[param_id]->mutable_gpu_data());
+
+ caffe_gpu_add_scalar(net_params[param_id]->count(),
+ delta, this->update_[param_id]->mutable_gpu_data());
+
+ caffe_gpu_div(net_params[param_id]->count(),
+ net_params[param_id]->gpu_diff(), this->update_[param_id]->gpu_data(),
+ this->update_[param_id]->mutable_gpu_data());
+
+ caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
+ this->update_[param_id]->gpu_data(), Dtype(0),
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+INSTANTIATE_CLASS(RMSPropSolver);
+REGISTER_SOLVER_CLASS(RMSProp);
+
+} // namespace caffe
diff --git a/src/caffe/solvers/sgd_solver.cpp b/src/caffe/solvers/sgd_solver.cpp
new file mode 100644
index 0000000..32bf19b
--- /dev/null
+++ b/src/caffe/solvers/sgd_solver.cpp
@@ -0,0 +1,348 @@
+#include <string>
+#include <vector>
+
+#include "caffe/sgd_solvers.hpp"
+#include "caffe/util/hdf5.hpp"
+#include "caffe/util/io.hpp"
+#include "caffe/util/upgrade_proto.hpp"
+
+namespace caffe {
+
+// Return the current learning rate. The currently implemented learning rate
+// policies are as follows:
+// - fixed: always return base_lr.
+// - step: return base_lr * gamma ^ (floor(iter / step))
+// - exp: return base_lr * gamma ^ iter
+// - inv: return base_lr * (1 + gamma * iter) ^ (- power)
+// - multistep: similar to step but it allows non uniform steps defined by
+// stepvalue
+// - poly: the effective learning rate follows a polynomial decay, to be
+// zero by the max_iter. return base_lr (1 - iter/max_iter) ^ (power)
+// - sigmoid: the effective learning rate follows a sigmod decay
+// return base_lr ( 1/(1 + exp(-gamma * (iter - stepsize))))
+//
+// where base_lr, max_iter, gamma, step, stepvalue and power are defined
+// in the solver parameter protocol buffer, and iter is the current iteration.
+template <typename Dtype>
+Dtype SGDSolver<Dtype>::GetLearningRate() {
+ Dtype rate;
+ const string& lr_policy = this->param_.lr_policy();
+ if (lr_policy == "fixed") {
+ rate = this->param_.base_lr();
+ } else if (lr_policy == "step") {
+ this->current_step_ = this->iter_ / this->param_.stepsize();
+ rate = this->param_.base_lr() *
+ pow(this->param_.gamma(), this->current_step_);
+ } else if (lr_policy == "exp") {
+ rate = this->param_.base_lr() * pow(this->param_.gamma(), this->iter_);
+ } else if (lr_policy == "inv") {
+ rate = this->param_.base_lr() *
+ pow(Dtype(1) + this->param_.gamma() * this->iter_,
+ - this->param_.power());
+ } else if (lr_policy == "multistep") {
+ if (this->current_step_ < this->param_.stepvalue_size() &&
+ this->iter_ >= this->param_.stepvalue(this->current_step_)) {
+ this->current_step_++;
+ LOG(INFO) << "MultiStep Status: Iteration " <<
+ this->iter_ << ", step = " << this->current_step_;
+ }
+ rate = this->param_.base_lr() *
+ pow(this->param_.gamma(), this->current_step_);
+ } else if (lr_policy == "poly") {
+ rate = this->param_.base_lr() * pow(Dtype(1.) -
+ (Dtype(this->iter_) / Dtype(this->param_.max_iter())),
+ this->param_.power());
+ } else if (lr_policy == "sigmoid") {
+ rate = this->param_.base_lr() * (Dtype(1.) /
+ (Dtype(1.) + exp(-this->param_.gamma() * (Dtype(this->iter_) -
+ Dtype(this->param_.stepsize())))));
+ } else {
+ LOG(FATAL) << "Unknown learning rate policy: " << lr_policy;
+ }
+ return rate;
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::PreSolve() {
+ // Initialize the history
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ history_.clear();
+ update_.clear();
+ temp_.clear();
+ for (int i = 0; i < net_params.size(); ++i) {
+ const vector<int>& shape = net_params[i]->shape();
+ history_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
+ update_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
+ temp_.push_back(shared_ptr<Blob<Dtype> >(new Blob<Dtype>(shape)));
+ }
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::ClipGradients() {
+ const Dtype clip_gradients = this->param_.clip_gradients();
+ if (clip_gradients < 0) { return; }
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ Dtype sumsq_diff = 0;
+ for (int i = 0; i < net_params.size(); ++i) {
+ sumsq_diff += net_params[i]->sumsq_diff();
+ }
+ const Dtype l2norm_diff = std::sqrt(sumsq_diff);
+ if (l2norm_diff > clip_gradients) {
+ Dtype scale_factor = clip_gradients / l2norm_diff;
+ LOG(INFO) << "Gradient clipping: scaling down gradients (L2 norm "
+ << l2norm_diff << " > " << clip_gradients << ") "
+ << "by scale factor " << scale_factor;
+ for (int i = 0; i < net_params.size(); ++i) {
+ net_params[i]->scale_diff(scale_factor);
+ }
+ }
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::ApplyUpdate() {
+ CHECK(Caffe::root_solver());
+ Dtype rate = GetLearningRate();
+ if (this->param_.display() && this->iter_ % this->param_.display() == 0) {
+ LOG(INFO) << "Iteration " << this->iter_ << ", lr = " << rate;
+ }
+ ClipGradients();
+ for (int param_id = 0; param_id < this->net_->learnable_params().size();
+ ++param_id) {
+ Normalize(param_id);
+ Regularize(param_id);
+ ComputeUpdateValue(param_id, rate);
+ }
+ this->net_->Update();
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::Normalize(int param_id) {
+ if (this->param_.iter_size() == 1) { return; }
+ // Scale gradient to counterbalance accumulation.
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const Dtype accum_normalization = Dtype(1.) / this->param_.iter_size();
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ caffe_scal(net_params[param_id]->count(), accum_normalization,
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ caffe_gpu_scal(net_params[param_id]->count(), accum_normalization,
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::Regularize(int param_id) {
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_weight_decay =
+ this->net_->params_weight_decay();
+ Dtype weight_decay = this->param_.weight_decay();
+ string regularization_type = this->param_.regularization_type();
+ Dtype local_decay = weight_decay * net_params_weight_decay[param_id];
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ if (local_decay) {
+ if (regularization_type == "L2") {
+ // add weight decay
+ caffe_axpy(net_params[param_id]->count(),
+ local_decay,
+ net_params[param_id]->cpu_data(),
+ net_params[param_id]->mutable_cpu_diff());
+ } else if (regularization_type == "L1") {
+ caffe_cpu_sign(net_params[param_id]->count(),
+ net_params[param_id]->cpu_data(),
+ temp_[param_id]->mutable_cpu_data());
+ caffe_axpy(net_params[param_id]->count(),
+ local_decay,
+ temp_[param_id]->cpu_data(),
+ net_params[param_id]->mutable_cpu_diff());
+ } else {
+ LOG(FATAL) << "Unknown regularization type: " << regularization_type;
+ }
+ }
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ if (local_decay) {
+ if (regularization_type == "L2") {
+ // add weight decay
+ caffe_gpu_axpy(net_params[param_id]->count(),
+ local_decay,
+ net_params[param_id]->gpu_data(),
+ net_params[param_id]->mutable_gpu_diff());
+ } else if (regularization_type == "L1") {
+ caffe_gpu_sign(net_params[param_id]->count(),
+ net_params[param_id]->gpu_data(),
+ temp_[param_id]->mutable_gpu_data());
+ caffe_gpu_axpy(net_params[param_id]->count(),
+ local_decay,
+ temp_[param_id]->gpu_data(),
+ net_params[param_id]->mutable_gpu_diff());
+ } else {
+ LOG(FATAL) << "Unknown regularization type: " << regularization_type;
+ }
+ }
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::ComputeUpdateValue(int param_id, Dtype rate) {
+ const vector<Blob<Dtype>*>& net_params = this->net_->learnable_params();
+ const vector<float>& net_params_lr = this->net_->params_lr();
+ Dtype momentum = this->param_.momentum();
+ Dtype local_rate = rate * net_params_lr[param_id];
+ // Compute the update to history, then copy it to the parameter diff.
+ switch (Caffe::mode()) {
+ case Caffe::CPU: {
+ caffe_cpu_axpby(net_params[param_id]->count(), local_rate,
+ net_params[param_id]->cpu_diff(), momentum,
+ history_[param_id]->mutable_cpu_data());
+ caffe_copy(net_params[param_id]->count(),
+ history_[param_id]->cpu_data(),
+ net_params[param_id]->mutable_cpu_diff());
+ break;
+ }
+ case Caffe::GPU: {
+#ifndef CPU_ONLY
+ caffe_gpu_axpby(net_params[param_id]->count(), local_rate,
+ net_params[param_id]->gpu_diff(), momentum,
+ history_[param_id]->mutable_gpu_data());
+ caffe_copy(net_params[param_id]->count(),
+ history_[param_id]->gpu_data(),
+ net_params[param_id]->mutable_gpu_diff());
+#else
+ NO_GPU;
+#endif
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode();
+ }
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::SnapshotSolverState(const string& model_filename) {
+ switch (this->param_.snapshot_format()) {
+ case caffe::SolverParameter_SnapshotFormat_BINARYPROTO:
+ SnapshotSolverStateToBinaryProto(model_filename);
+ break;
+ case caffe::SolverParameter_SnapshotFormat_HDF5:
+ SnapshotSolverStateToHDF5(model_filename);
+ break;
+ default:
+ LOG(FATAL) << "Unsupported snapshot format.";
+ }
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::SnapshotSolverStateToBinaryProto(
+ const string& model_filename) {
+ SolverState state;
+ state.set_iter(this->iter_);
+ state.set_learned_net(model_filename);
+ state.set_current_step(this->current_step_);
+ state.clear_history();
+ for (int i = 0; i < history_.size(); ++i) {
+ // Add history
+ BlobProto* history_blob = state.add_history();
+ history_[i]->ToProto(history_blob);
+ }
+ string snapshot_filename = Solver<Dtype>::SnapshotFilename(".solverstate");
+ LOG(INFO)
+ << "Snapshotting solver state to binary proto file " << snapshot_filename;
+ WriteProtoToBinaryFile(state, snapshot_filename.c_str());
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::SnapshotSolverStateToHDF5(
+ const string& model_filename) {
+ string snapshot_filename =
+ Solver<Dtype>::SnapshotFilename(".solverstate.h5");
+ LOG(INFO) << "Snapshotting solver state to HDF5 file " << snapshot_filename;
+ hid_t file_hid = H5Fcreate(snapshot_filename.c_str(), H5F_ACC_TRUNC,
+ H5P_DEFAULT, H5P_DEFAULT);
+ CHECK_GE(file_hid, 0)
+ << "Couldn't open " << snapshot_filename << " to save solver state.";
+ hdf5_save_int(file_hid, "iter", this->iter_);
+ hdf5_save_string(file_hid, "learned_net", model_filename);
+ hdf5_save_int(file_hid, "current_step", this->current_step_);
+ hid_t history_hid = H5Gcreate2(file_hid, "history", H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT);
+ CHECK_GE(history_hid, 0)
+ << "Error saving solver state to " << snapshot_filename << ".";
+ for (int i = 0; i < history_.size(); ++i) {
+ ostringstream oss;
+ oss << i;
+ hdf5_save_nd_dataset<Dtype>(history_hid, oss.str(), *history_[i]);
+ }
+ H5Gclose(history_hid);
+ H5Fclose(file_hid);
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::RestoreSolverStateFromBinaryProto(
+ const string& state_file) {
+ SolverState state;
+ ReadProtoFromBinaryFile(state_file, &state);
+ this->iter_ = state.iter();
+ if (state.has_learned_net()) {
+ NetParameter net_param;
+ ReadNetParamsFromBinaryFileOrDie(state.learned_net().c_str(), &net_param);
+ this->net_->CopyTrainedLayersFrom(net_param);
+ }
+ this->current_step_ = state.current_step();
+ CHECK_EQ(state.history_size(), history_.size())
+ << "Incorrect length of history blobs.";
+ LOG(INFO) << "SGDSolver: restoring history";
+ for (int i = 0; i < history_.size(); ++i) {
+ history_[i]->FromProto(state.history(i));
+ }
+}
+
+template <typename Dtype>
+void SGDSolver<Dtype>::RestoreSolverStateFromHDF5(const string& state_file) {
+ hid_t file_hid = H5Fopen(state_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
+ CHECK_GE(file_hid, 0) << "Couldn't open solver state file " << state_file;
+ this->iter_ = hdf5_load_int(file_hid, "iter");
+ if (H5LTfind_dataset(file_hid, "learned_net")) {
+ string learned_net = hdf5_load_string(file_hid, "learned_net");
+ this->net_->CopyTrainedLayersFrom(learned_net);
+ }
+ this->current_step_ = hdf5_load_int(file_hid, "current_step");
+ hid_t history_hid = H5Gopen2(file_hid, "history", H5P_DEFAULT);
+ CHECK_GE(history_hid, 0) << "Error reading history from " << state_file;
+ int state_history_size = hdf5_get_num_links(history_hid);
+ CHECK_EQ(state_history_size, history_.size())
+ << "Incorrect length of history blobs.";
+ for (int i = 0; i < history_.size(); ++i) {
+ ostringstream oss;
+ oss << i;
+ hdf5_load_nd_dataset<Dtype>(history_hid, oss.str().c_str(), 0,
+ kMaxBlobAxes, history_[i].get());
+ }
+ H5Gclose(history_hid);
+ H5Fclose(file_hid);
+}
+
+INSTANTIATE_CLASS(SGDSolver);
+REGISTER_SOLVER_CLASS(SGD);
+
+} // namespace caffe
diff --git a/src/caffe/syncedmem.cpp b/src/caffe/syncedmem.cpp
index a667a86..4d35641 100644
--- a/src/caffe/syncedmem.cpp
+++ b/src/caffe/syncedmem.cpp
@@ -1,5 +1,3 @@
-#include <cstring>
-
#include "caffe/common.hpp"
#include "caffe/syncedmem.hpp"
#include "caffe/util/math_functions.hpp"
@@ -8,7 +6,7 @@ namespace caffe {
SyncedMemory::~SyncedMemory() {
if (cpu_ptr_ && own_cpu_data_) {
- CaffeFreeHost(cpu_ptr_);
+ CaffeFreeHost(cpu_ptr_, cpu_malloc_use_cuda_);
}
#ifndef CPU_ONLY
@@ -27,7 +25,7 @@ SyncedMemory::~SyncedMemory() {
inline void SyncedMemory::to_cpu() {
switch (head_) {
case UNINITIALIZED:
- CaffeMallocHost(&cpu_ptr_, size_);
+ CaffeMallocHost(&cpu_ptr_, size_, &cpu_malloc_use_cuda_);
caffe_memset(size_, 0, cpu_ptr_);
head_ = HEAD_AT_CPU;
own_cpu_data_ = true;
@@ -35,7 +33,7 @@ inline void SyncedMemory::to_cpu() {
case HEAD_AT_GPU:
#ifndef CPU_ONLY
if (cpu_ptr_ == NULL) {
- CaffeMallocHost(&cpu_ptr_, size_);
+ CaffeMallocHost(&cpu_ptr_, size_, &cpu_malloc_use_cuda_);
own_cpu_data_ = true;
}
caffe_gpu_memcpy(size_, gpu_ptr_, cpu_ptr_);
@@ -86,7 +84,7 @@ const void* SyncedMemory::cpu_data() {
void SyncedMemory::set_cpu_data(void* data) {
CHECK(data);
if (own_cpu_data_) {
- CaffeFreeHost(cpu_ptr_);
+ CaffeFreeHost(cpu_ptr_, cpu_malloc_use_cuda_);
}
cpu_ptr_ = data;
head_ = HEAD_AT_CPU;
@@ -99,6 +97,7 @@ const void* SyncedMemory::gpu_data() {
return (const void*)gpu_ptr_;
#else
NO_GPU;
+ return NULL;
#endif
}
@@ -135,6 +134,7 @@ void* SyncedMemory::mutable_gpu_data() {
return gpu_ptr_;
#else
NO_GPU;
+ return NULL;
#endif
}
diff --git a/src/caffe/test/test_accuracy_layer.cpp b/src/caffe/test/test_accuracy_layer.cpp
index ef0e57a..6fe808b 100644
--- a/src/caffe/test/test_accuracy_layer.cpp
+++ b/src/caffe/test/test_accuracy_layer.cpp
@@ -1,6 +1,4 @@
#include <cfloat>
-#include <cmath>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -8,8 +6,8 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/accuracy_layer.hpp"
#include "caffe/util/rng.hpp"
-#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_argmax_layer.cpp b/src/caffe/test/test_argmax_layer.cpp
index 895c3d3..472e665 100644
--- a/src/caffe/test/test_argmax_layer.cpp
+++ b/src/caffe/test/test_argmax_layer.cpp
@@ -6,7 +6,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/argmax_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
@@ -16,7 +16,7 @@ template <typename Dtype>
class ArgMaxLayerTest : public CPUDeviceTest<Dtype> {
protected:
ArgMaxLayerTest()
- : blob_bottom_(new Blob<Dtype>(10, 20, 1, 1)),
+ : blob_bottom_(new Blob<Dtype>(10, 10, 20, 20)),
blob_top_(new Blob<Dtype>()),
top_k_(5) {
Caffe::set_random_seed(1701);
@@ -55,6 +55,43 @@ TYPED_TEST(ArgMaxLayerTest, TestSetupMaxVal) {
EXPECT_EQ(this->blob_top_->channels(), 2);
}
+TYPED_TEST(ArgMaxLayerTest, TestSetupAxis) {
+ LayerParameter layer_param;
+ ArgMaxParameter* argmax_param = layer_param.mutable_argmax_param();
+ argmax_param->set_axis(0);
+ ArgMaxLayer<TypeParam> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ EXPECT_EQ(this->blob_top_->shape(0), argmax_param->top_k());
+ EXPECT_EQ(this->blob_top_->shape(1), this->blob_bottom_->shape(0));
+ EXPECT_EQ(this->blob_top_->shape(2), this->blob_bottom_->shape(2));
+ EXPECT_EQ(this->blob_top_->shape(3), this->blob_bottom_->shape(3));
+}
+
+TYPED_TEST(ArgMaxLayerTest, TestSetupAxisNegativeIndexing) {
+ LayerParameter layer_param;
+ ArgMaxParameter* argmax_param = layer_param.mutable_argmax_param();
+ argmax_param->set_axis(-2);
+ ArgMaxLayer<TypeParam> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ EXPECT_EQ(this->blob_top_->shape(0), this->blob_bottom_->shape(0));
+ EXPECT_EQ(this->blob_top_->shape(1), this->blob_bottom_->shape(1));
+ EXPECT_EQ(this->blob_top_->shape(2), argmax_param->top_k());
+ EXPECT_EQ(this->blob_top_->shape(3), this->blob_bottom_->shape(3));
+}
+
+TYPED_TEST(ArgMaxLayerTest, TestSetupAxisMaxVal) {
+ LayerParameter layer_param;
+ ArgMaxParameter* argmax_param = layer_param.mutable_argmax_param();
+ argmax_param->set_axis(2);
+ argmax_param->set_out_max_val(true);
+ ArgMaxLayer<TypeParam> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ EXPECT_EQ(this->blob_top_->shape(0), this->blob_bottom_->shape(0));
+ EXPECT_EQ(this->blob_top_->shape(1), this->blob_bottom_->shape(1));
+ EXPECT_EQ(this->blob_top_->shape(2), argmax_param->top_k());
+ EXPECT_EQ(this->blob_top_->shape(3), this->blob_bottom_->shape(3));
+}
+
TYPED_TEST(ArgMaxLayerTest, TestCPU) {
LayerParameter layer_param;
ArgMaxLayer<TypeParam> layer(layer_param);
@@ -112,6 +149,7 @@ TYPED_TEST(ArgMaxLayerTest, TestCPUTopK) {
layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
// Now, check values
+ const TypeParam* bottom_data = this->blob_bottom_->cpu_data();
int max_ind;
TypeParam max_val;
int num = this->blob_bottom_->num();
@@ -121,10 +159,10 @@ TYPED_TEST(ArgMaxLayerTest, TestCPUTopK) {
EXPECT_LE(this->blob_top_->data_at(i, 0, 0, 0), dim);
for (int j = 0; j < this->top_k_; ++j) {
max_ind = this->blob_top_->data_at(i, 0, j, 0);
- max_val = this->blob_bottom_->data_at(i, max_ind, 0, 0);
+ max_val = bottom_data[i * dim + max_ind];
int count = 0;
for (int k = 0; k < dim; ++k) {
- if (this->blob_bottom_->data_at(i, k, 0, 0) > max_val) {
+ if (bottom_data[i * dim + k] > max_val) {
++count;
}
}
@@ -142,6 +180,7 @@ TYPED_TEST(ArgMaxLayerTest, TestCPUMaxValTopK) {
layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
// Now, check values
+ const TypeParam* bottom_data = this->blob_bottom_->cpu_data();
int max_ind;
TypeParam max_val;
int num = this->blob_bottom_->num();
@@ -152,10 +191,10 @@ TYPED_TEST(ArgMaxLayerTest, TestCPUMaxValTopK) {
for (int j = 0; j < this->top_k_; ++j) {
max_ind = this->blob_top_->data_at(i, 0, j, 0);
max_val = this->blob_top_->data_at(i, 1, j, 0);
- EXPECT_EQ(this->blob_bottom_->data_at(i, max_ind, 0, 0), max_val);
+ EXPECT_EQ(bottom_data[i * dim + max_ind], max_val);
int count = 0;
for (int k = 0; k < dim; ++k) {
- if (this->blob_bottom_->data_at(i, k, 0, 0) > max_val) {
+ if (bottom_data[i * dim + k] > max_val) {
++count;
}
}
@@ -164,5 +203,93 @@ TYPED_TEST(ArgMaxLayerTest, TestCPUMaxValTopK) {
}
}
+TYPED_TEST(ArgMaxLayerTest, TestCPUAxis) {
+ LayerParameter layer_param;
+ ArgMaxParameter* argmax_param = layer_param.mutable_argmax_param();
+ argmax_param->set_axis(0);
+ ArgMaxLayer<TypeParam> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ // Now, check values
+ int max_ind;
+ TypeParam max_val;
+ std::vector<int> shape = this->blob_bottom_->shape();
+ for (int i = 0; i < shape[1]; ++i) {
+ for (int j = 0; j < shape[2]; ++j) {
+ for (int k = 0; k < shape[3]; ++k) {
+ max_ind = this->blob_top_->data_at(0, i, j, k);
+ max_val = this->blob_bottom_->data_at(max_ind, i, j, k);
+ EXPECT_GE(max_ind, 0);
+ EXPECT_LE(max_ind, shape[0]);
+ for (int l = 0; l < shape[0]; ++l) {
+ EXPECT_LE(this->blob_bottom_->data_at(l, i, j, k), max_val);
+ }
+ }
+ }
+ }
+}
+
+TYPED_TEST(ArgMaxLayerTest, TestCPUAxisTopK) {
+ LayerParameter layer_param;
+ ArgMaxParameter* argmax_param = layer_param.mutable_argmax_param();
+ argmax_param->set_axis(2);
+ argmax_param->set_top_k(this->top_k_);
+ ArgMaxLayer<TypeParam> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ // Now, check values
+ int max_ind;
+ TypeParam max_val;
+ std::vector<int> shape = this->blob_bottom_->shape();
+ for (int i = 0; i < shape[0]; ++i) {
+ for (int j = 0; j < shape[1]; ++j) {
+ for (int k = 0; k < shape[3]; ++k) {
+ for (int m = 0; m < this->top_k_; ++m) {
+ max_ind = this->blob_top_->data_at(i, j, m, k);
+ max_val = this->blob_bottom_->data_at(i, j, max_ind, k);
+ EXPECT_GE(max_ind, 0);
+ EXPECT_LE(max_ind, shape[2]);
+ int count = 0;
+ for (int l = 0; l < shape[2]; ++l) {
+ if (this->blob_bottom_->data_at(i, j, l, k) > max_val) {
+ ++count;
+ }
+ }
+ EXPECT_EQ(m, count);
+ }
+ }
+ }
+ }
+}
+
+TYPED_TEST(ArgMaxLayerTest, TestCPUAxisMaxValTopK) {
+ LayerParameter layer_param;
+ ArgMaxParameter* argmax_param = layer_param.mutable_argmax_param();
+ argmax_param->set_axis(-1);
+ argmax_param->set_top_k(this->top_k_);
+ argmax_param->set_out_max_val(true);
+ ArgMaxLayer<TypeParam> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ // Now, check values
+ TypeParam max_val;
+ std::vector<int> shape = this->blob_bottom_->shape();
+ for (int i = 0; i < shape[0]; ++i) {
+ for (int j = 0; j < shape[1]; ++j) {
+ for (int k = 0; k < shape[2]; ++k) {
+ for (int m = 0; m < this->top_k_; ++m) {
+ max_val = this->blob_top_->data_at(i, j, k, m);
+ int count = 0;
+ for (int l = 0; l < shape[3]; ++l) {
+ if (this->blob_bottom_->data_at(i, j, k, l) > max_val) {
+ ++count;
+ }
+ }
+ EXPECT_EQ(m, count);
+ }
+ }
+ }
+ }
+}
} // namespace caffe
diff --git a/src/caffe/test/test_batch_norm_layer.cpp b/src/caffe/test/test_batch_norm_layer.cpp
new file mode 100644
index 0000000..936b93a
--- /dev/null
+++ b/src/caffe/test/test_batch_norm_layer.cpp
@@ -0,0 +1,133 @@
+#include <algorithm>
+#include <cstring>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "caffe/blob.hpp"
+#include "caffe/common.hpp"
+#include "caffe/filler.hpp"
+#include "caffe/layers/batch_norm_layer.hpp"
+
+#include "caffe/test/test_caffe_main.hpp"
+#include "caffe/test/test_gradient_check_util.hpp"
+
+#define BATCH_SIZE 2
+#define INPUT_DATA_SIZE 3
+
+namespace caffe {
+
+ template <typename TypeParam>
+ class BatchNormLayerTest : public MultiDeviceTest<TypeParam> {
+ typedef typename TypeParam::Dtype Dtype;
+ protected:
+ BatchNormLayerTest()
+ : blob_bottom_(new Blob<Dtype>(5, 2, 3, 4)),
+ blob_top_(new Blob<Dtype>()) {
+ // fill the values
+ FillerParameter filler_param;
+ GaussianFiller<Dtype> filler(filler_param);
+ filler.Fill(this->blob_bottom_);
+ blob_bottom_vec_.push_back(blob_bottom_);
+ blob_top_vec_.push_back(blob_top_);
+ }
+ virtual ~BatchNormLayerTest() { delete blob_bottom_; delete blob_top_; }
+ Blob<Dtype>* const blob_bottom_;
+ Blob<Dtype>* const blob_top_;
+ vector<Blob<Dtype>*> blob_bottom_vec_;
+ vector<Blob<Dtype>*> blob_top_vec_;
+ };
+
+ TYPED_TEST_CASE(BatchNormLayerTest, TestDtypesAndDevices);
+
+ TYPED_TEST(BatchNormLayerTest, TestForward) {
+ typedef typename TypeParam::Dtype Dtype;
+ LayerParameter layer_param;
+
+ BatchNormLayer<Dtype> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+
+ // Test mean
+ int num = this->blob_bottom_->num();
+ int channels = this->blob_bottom_->channels();
+ int height = this->blob_bottom_->height();
+ int width = this->blob_bottom_->width();
+
+ for (int j = 0; j < channels; ++j) {
+ Dtype sum = 0, var = 0;
+ for (int i = 0; i < num; ++i) {
+ for ( int k = 0; k < height; ++k ) {
+ for ( int l = 0; l < width; ++l ) {
+ Dtype data = this->blob_top_->data_at(i, j, k, l);
+ sum += data;
+ var += data * data;
+ }
+ }
+ }
+ sum /= height * width * num;
+ var /= height * width * num;
+
+ const Dtype kErrorBound = 0.001;
+ // expect zero mean
+ EXPECT_NEAR(0, sum, kErrorBound);
+ // expect unit variance
+ EXPECT_NEAR(1, var, kErrorBound);
+ }
+ }
+
+ TYPED_TEST(BatchNormLayerTest, TestForwardInplace) {
+ typedef typename TypeParam::Dtype Dtype;
+ Blob<Dtype> blob_inplace(5, 2, 3, 4);
+ vector<Blob<Dtype>*> blob_bottom_vec;
+ vector<Blob<Dtype>*> blob_top_vec;
+ LayerParameter layer_param;
+ FillerParameter filler_param;
+ GaussianFiller<Dtype> filler(filler_param);
+ filler.Fill(&blob_inplace);
+ blob_bottom_vec.push_back(&blob_inplace);
+ blob_top_vec.push_back(&blob_inplace);
+
+ BatchNormLayer<Dtype> layer(layer_param);
+ layer.SetUp(blob_bottom_vec, blob_top_vec);
+ layer.Forward(blob_bottom_vec, blob_top_vec);
+
+ // Test mean
+ int num = blob_inplace.num();
+ int channels = blob_inplace.channels();
+ int height = blob_inplace.height();
+ int width = blob_inplace.width();
+
+ for (int j = 0; j < channels; ++j) {
+ Dtype sum = 0, var = 0;
+ for (int i = 0; i < num; ++i) {
+ for ( int k = 0; k < height; ++k ) {
+ for ( int l = 0; l < width; ++l ) {
+ Dtype data = blob_inplace.data_at(i, j, k, l);
+ sum += data;
+ var += data * data;
+ }
+ }
+ }
+ sum /= height * width * num;
+ var /= height * width * num;
+
+ const Dtype kErrorBound = 0.001;
+ // expect zero mean
+ EXPECT_NEAR(0, sum, kErrorBound);
+ // expect unit variance
+ EXPECT_NEAR(1, var, kErrorBound);
+ }
+ }
+
+ TYPED_TEST(BatchNormLayerTest, TestGradient) {
+ typedef typename TypeParam::Dtype Dtype;
+ LayerParameter layer_param;
+
+ BatchNormLayer<Dtype> layer(layer_param);
+ GradientChecker<Dtype> checker(1e-2, 1e-4);
+ checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_,
+ this->blob_top_vec_);
+ }
+
+} // namespace caffe
diff --git a/src/caffe/test/test_batch_reindex_layer.cpp b/src/caffe/test/test_batch_reindex_layer.cpp
new file mode 100644
index 0000000..9ea1a2f
--- /dev/null
+++ b/src/caffe/test/test_batch_reindex_layer.cpp
@@ -0,0 +1,118 @@
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "caffe/blob.hpp"
+#include "caffe/common.hpp"
+#include "caffe/filler.hpp"
+#include "caffe/layers/batch_reindex_layer.hpp"
+
+#include "caffe/test/test_caffe_main.hpp"
+#include "caffe/test/test_gradient_check_util.hpp"
+
+namespace caffe {
+
+template<typename TypeParam>
+class BatchReindexLayerTest : public MultiDeviceTest<TypeParam> {
+ typedef typename TypeParam::Dtype Dtype;
+
+ protected:
+ BatchReindexLayerTest()
+ : blob_bottom_(new Blob<Dtype>()),
+ blob_bottom_permute_(new Blob<Dtype>()),
+ blob_top_(new Blob<Dtype>()) {
+ }
+ virtual void SetUp() {
+ Caffe::set_random_seed(1701);
+ vector<int> sz;
+ sz.push_back(5);
+ sz.push_back(4);
+ sz.push_back(3);
+ sz.push_back(2);
+ blob_bottom_->Reshape(sz);
+ vector<int> permsz;
+ permsz.push_back(6);
+ blob_bottom_permute_->Reshape(permsz);
+
+ // fill the values
+ FillerParameter filler_param;
+ GaussianFiller<Dtype> filler(filler_param);
+ filler.Fill(this->blob_bottom_);
+ int perm[] = { 4, 0, 4, 0, 1, 2 };
+ for (int i = 0; i < blob_bottom_permute_->count(); ++i) {
+ blob_bottom_permute_->mutable_cpu_data()[i] = perm[i];
+ }
+
+ blob_bottom_vec_.push_back(blob_bottom_);
+ blob_bottom_vec_.push_back(blob_bottom_permute_);
+ blob_top_vec_.push_back(blob_top_);
+ }
+ virtual ~BatchReindexLayerTest() {
+ delete blob_bottom_permute_;
+ delete blob_bottom_;
+ delete blob_top_;
+ }
+ Blob<Dtype>* const blob_bottom_;
+ Blob<Dtype>* const blob_bottom_permute_;
+ Blob<Dtype>* const blob_top_;
+ vector<Blob<Dtype>*> blob_bottom_vec_;
+ vector<Blob<Dtype>*> blob_top_vec_;
+
+ void TestForward() {
+ LayerParameter layer_param;
+
+ vector<int> sz;
+ sz.push_back(5);
+ sz.push_back(4);
+ sz.push_back(3);
+ sz.push_back(2);
+ blob_bottom_->Reshape(sz);
+ for (int i = 0; i < blob_bottom_->count(); ++i) {
+ blob_bottom_->mutable_cpu_data()[i] = i;
+ }
+
+ vector<int> permsz;
+ permsz.push_back(6);
+ blob_bottom_permute_->Reshape(permsz);
+ int perm[] = { 4, 0, 4, 0, 1, 2 };
+ for (int i = 0; i < blob_bottom_permute_->count(); ++i) {
+ blob_bottom_permute_->mutable_cpu_data()[i] = perm[i];
+ }
+ BatchReindexLayer<Dtype> layer(layer_param);
+ layer.SetUp(blob_bottom_vec_, blob_top_vec_);
+ EXPECT_EQ(blob_top_->num(), blob_bottom_permute_->num());
+ EXPECT_EQ(blob_top_->channels(), blob_bottom_->channels());
+ EXPECT_EQ(blob_top_->height(), blob_bottom_->height());
+ EXPECT_EQ(blob_top_->width(), blob_bottom_->width());
+
+ layer.Forward(blob_bottom_vec_, blob_top_vec_);
+ int channels = blob_top_->channels();
+ int height = blob_top_->height();
+ int width = blob_top_->width();
+ for (int i = 0; i < blob_top_->count(); ++i) {
+ int n = i / (channels * width * height);
+ int inner_idx = (i % (channels * width * height));
+ EXPECT_EQ(
+ blob_top_->cpu_data()[i],
+ blob_bottom_->cpu_data()[perm[n] * channels * width * height
+ + inner_idx]);
+ }
+ }
+};
+
+TYPED_TEST_CASE(BatchReindexLayerTest, TestDtypesAndDevices);
+
+TYPED_TEST(BatchReindexLayerTest, TestForward) {
+ this->TestForward();
+}
+
+TYPED_TEST(BatchReindexLayerTest, TestGradient) {
+ typedef typename TypeParam::Dtype Dtype;
+ LayerParameter layer_param;
+ BatchReindexLayer<Dtype> layer(layer_param);
+ GradientChecker<Dtype> checker(1e-4, 1e-2);
+ checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_,
+ this->blob_top_vec_, 0);
+ }
+
+} // namespace caffe
diff --git a/src/caffe/test/test_benchmark.cpp b/src/caffe/test/test_benchmark.cpp
index 43aaa63..b03fdf6 100644
--- a/src/caffe/test/test_benchmark.cpp
+++ b/src/caffe/test/test_benchmark.cpp
@@ -1,4 +1,4 @@
-#include <unistd.h> // for usleep
+#include <boost/thread.hpp>
#include "gtest/gtest.h"
@@ -64,7 +64,7 @@ TYPED_TEST(BenchmarkTest, TestTimerMilliSeconds) {
EXPECT_FALSE(timer.running());
EXPECT_FALSE(timer.has_run_at_least_once());
timer.Start();
- usleep(300 * 1000);
+ boost::this_thread::sleep(boost::posix_time::milliseconds(300));
EXPECT_GE(timer.MilliSeconds(), 300 - kMillisecondsThreshold);
EXPECT_LE(timer.MilliSeconds(), 300 + kMillisecondsThreshold);
EXPECT_TRUE(timer.initted());
@@ -79,7 +79,7 @@ TYPED_TEST(BenchmarkTest, TestTimerSeconds) {
EXPECT_FALSE(timer.running());
EXPECT_FALSE(timer.has_run_at_least_once());
timer.Start();
- usleep(300 * 1000);
+ boost::this_thread::sleep(boost::posix_time::milliseconds(300));
EXPECT_GE(timer.Seconds(), 0.3 - kMillisecondsThreshold / 1000.);
EXPECT_LE(timer.Seconds(), 0.3 + kMillisecondsThreshold / 1000.);
EXPECT_TRUE(timer.initted());
diff --git a/src/caffe/test/test_blob.cpp b/src/caffe/test/test_blob.cpp
index 7da6423..a9d7d51 100644
--- a/src/caffe/test/test_blob.cpp
+++ b/src/caffe/test/test_blob.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
diff --git a/src/caffe/test/test_common.cpp b/src/caffe/test/test_common.cpp
index b3a61b0..58ae5c6 100644
--- a/src/caffe/test/test_common.cpp
+++ b/src/caffe/test/test_common.cpp
@@ -1,5 +1,3 @@
-#include <cstring>
-
#include "gtest/gtest.h"
#include "caffe/common.hpp"
diff --git a/src/caffe/test/test_concat_layer.cpp b/src/caffe/test/test_concat_layer.cpp
index ccd97eb..23c1e8c 100644
--- a/src/caffe/test/test_concat_layer.cpp
+++ b/src/caffe/test/test_concat_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/concat_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_contrastive_loss_layer.cpp b/src/caffe/test/test_contrastive_loss_layer.cpp
index 1e9447c..2fa055e 100644
--- a/src/caffe/test/test_contrastive_loss_layer.cpp
+++ b/src/caffe/test/test_contrastive_loss_layer.cpp
@@ -1,7 +1,5 @@
#include <algorithm>
#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -9,7 +7,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/contrastive_loss_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
@@ -79,7 +77,7 @@ TYPED_TEST(ContrastiveLossLayerTest, TestForward) {
if (this->blob_bottom_y_->cpu_data()[i]) { // similar pairs
loss += dist_sq;
} else {
- Dtype dist = std::max(margin - sqrt(dist_sq), 0.0);
+ Dtype dist = std::max<Dtype>(margin - sqrt(dist_sq), 0.0);
loss += dist*dist;
}
}
diff --git a/src/caffe/test/test_convolution_layer.cpp b/src/caffe/test/test_convolution_layer.cpp
index 9df979a..e2d43f3 100644
--- a/src/caffe/test/test_convolution_layer.cpp
+++ b/src/caffe/test/test_convolution_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,11 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/conv_layer.hpp"
+
+#ifdef USE_CUDNN
+#include "caffe/layers/cudnn_conv_layer.hpp"
+#endif
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_data_layer.cpp b/src/caffe/test/test_data_layer.cpp
index 9e03954..3e8d113 100644
--- a/src/caffe/test/test_data_layer.cpp
+++ b/src/caffe/test/test_data_layer.cpp
@@ -7,8 +7,8 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
-#include "caffe/data_layers.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/db.hpp"
#include "caffe/util/io.hpp"
diff --git a/src/caffe/test/test_deconvolution_layer.cpp b/src/caffe/test/test_deconvolution_layer.cpp
index 770e7b2..c4b09ad 100644
--- a/src/caffe/test/test_deconvolution_layer.cpp
+++ b/src/caffe/test/test_deconvolution_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/deconv_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_dummy_data_layer.cpp b/src/caffe/test/test_dummy_data_layer.cpp
index c9ed38d..1a01ca8 100644
--- a/src/caffe/test/test_dummy_data_layer.cpp
+++ b/src/caffe/test/test_dummy_data_layer.cpp
@@ -5,8 +5,8 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
+#include "caffe/layers/dummy_data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
-#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_eltwise_layer.cpp b/src/caffe/test/test_eltwise_layer.cpp
index 8031f6e..c06e3ba 100644
--- a/src/caffe/test/test_eltwise_layer.cpp
+++ b/src/caffe/test/test_eltwise_layer.cpp
@@ -6,7 +6,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/eltwise_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_embed_layer.cpp b/src/caffe/test/test_embed_layer.cpp
index 7a4fb98..acd4b0f 100644
--- a/src/caffe/test/test_embed_layer.cpp
+++ b/src/caffe/test/test_embed_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/embed_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_euclidean_loss_layer.cpp b/src/caffe/test/test_euclidean_loss_layer.cpp
index 1949742..f253f9f 100644
--- a/src/caffe/test/test_euclidean_loss_layer.cpp
+++ b/src/caffe/test/test_euclidean_loss_layer.cpp
@@ -1,6 +1,4 @@
#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -8,7 +6,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/euclidean_loss_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_filler.cpp b/src/caffe/test/test_filler.cpp
index 728b8dc..26e9b21 100644
--- a/src/caffe/test/test_filler.cpp
+++ b/src/caffe/test/test_filler.cpp
@@ -1,5 +1,3 @@
-#include <cstring>
-
#include "gtest/gtest.h"
#include "caffe/filler.hpp"
diff --git a/src/caffe/test/test_filter_layer.cpp b/src/caffe/test/test_filter_layer.cpp
index c641b6e..9ea2b8b 100644
--- a/src/caffe/test/test_filter_layer.cpp
+++ b/src/caffe/test/test_filter_layer.cpp
@@ -1,5 +1,3 @@
-#include <cstring>
-#include <limits>
#include <vector>
#include "gtest/gtest.h"
@@ -7,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/filter_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_flatten_layer.cpp b/src/caffe/test/test_flatten_layer.cpp
index 7b6757c..d929ac7 100644
--- a/src/caffe/test/test_flatten_layer.cpp
+++ b/src/caffe/test/test_flatten_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/flatten_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_gradient_based_solver.cpp b/src/caffe/test/test_gradient_based_solver.cpp
index 7ad7467..84c6747 100644
--- a/src/caffe/test/test_gradient_based_solver.cpp
+++ b/src/caffe/test/test_gradient_based_solver.cpp
@@ -10,7 +10,7 @@
#include "caffe/common.hpp"
#include "caffe/parallel.hpp"
#include "caffe/proto/caffe.pb.h"
-#include "caffe/solver.hpp"
+#include "caffe/sgd_solvers.hpp"
#include "caffe/util/io.hpp"
#include "caffe/test/test_caffe_main.hpp"
@@ -47,7 +47,6 @@ class GradientBasedSolverTest : public MultiDeviceTest<TypeParam> {
// Test data: check out generate_sample_data.py in the same directory.
string* input_file_;
- virtual SolverParameter_SolverType solver_type() = 0;
virtual void InitSolver(const SolverParameter& param) = 0;
virtual void InitSolverFromProtoString(const string& proto) {
@@ -290,8 +289,8 @@ class GradientBasedSolverTest : public MultiDeviceTest<TypeParam> {
((i == D) ? bias.cpu_data()[0] : weights.cpu_data()[i]);
// Finally, compute update.
const vector<shared_ptr<Blob<Dtype> > >& history = solver_->history();
- if (solver_type() != SolverParameter_SolverType_ADADELTA
- && solver_type() != SolverParameter_SolverType_ADAM) {
+ if (solver_->type() != string("AdaDelta")
+ && solver_->type() != string("Adam")) {
ASSERT_EQ(2, history.size()); // 1 blob for weights, 1 for bias
} else {
ASSERT_EQ(4, history.size()); // additional blobs for update history
@@ -300,26 +299,19 @@ class GradientBasedSolverTest : public MultiDeviceTest<TypeParam> {
const Dtype history_value = (i == D) ?
history[1]->cpu_data()[0] : history[0]->cpu_data()[i];
const Dtype temp = momentum * history_value;
- switch (solver_type()) {
- case SolverParameter_SolverType_SGD:
+ if (solver_->type() == string("SGD")) {
update_value += temp;
- break;
- case SolverParameter_SolverType_NESTEROV:
+ } else if (solver_->type() == string("Nesterov")) {
update_value += temp;
// step back then over-step
update_value = (1 + momentum) * update_value - temp;
- break;
- case SolverParameter_SolverType_ADAGRAD:
+ } else if (solver_->type() == string("AdaGrad")) {
update_value /= std::sqrt(history_value + grad * grad) + delta_;
- break;
- case SolverParameter_SolverType_RMSPROP: {
+ } else if (solver_->type() == string("RMSProp")) {
const Dtype rms_decay = 0.95;
update_value /= std::sqrt(rms_decay*history_value
+ grad * grad * (1 - rms_decay)) + delta_;
- }
- break;
- case SolverParameter_SolverType_ADADELTA:
- {
+ } else if (solver_->type() == string("AdaDelta")) {
const Dtype update_history_value = (i == D) ?
history[1 + num_param_blobs]->cpu_data()[0] :
history[0 + num_param_blobs]->cpu_data()[i];
@@ -330,9 +322,7 @@ class GradientBasedSolverTest : public MultiDeviceTest<TypeParam> {
// not actually needed, just here for illustrative purposes
// const Dtype weighted_update_average =
// momentum * update_history_value + (1 - momentum) * (update_value);
- break;
- }
- case SolverParameter_SolverType_ADAM: {
+ } else if (solver_->type() == string("Adam")) {
const Dtype momentum2 = 0.999;
const Dtype m = history_value;
const Dtype v = (i == D) ?
@@ -344,10 +334,8 @@ class GradientBasedSolverTest : public MultiDeviceTest<TypeParam> {
std::sqrt(Dtype(1) - pow(momentum2, num_iters)) /
(Dtype(1.) - pow(momentum, num_iters));
update_value = alpha_t * val_m / (std::sqrt(val_v) + delta_);
- break;
- }
- default:
- LOG(FATAL) << "Unknown solver type: " << solver_type();
+ } else {
+ LOG(FATAL) << "Unknown solver type: " << solver_->type();
}
if (i == D) {
updated_bias.mutable_cpu_diff()[0] = update_value;
@@ -392,7 +380,7 @@ class GradientBasedSolverTest : public MultiDeviceTest<TypeParam> {
EXPECT_NEAR(expected_updated_bias, solver_updated_bias, error_margin);
// Check the solver's history -- should contain the previous update value.
- if (solver_type() == SolverParameter_SolverType_SGD) {
+ if (solver_->type() == string("SGD")) {
const vector<shared_ptr<Blob<Dtype> > >& history = solver_->history();
ASSERT_EQ(2, history.size());
for (int i = 0; i < D; ++i) {
@@ -581,10 +569,6 @@ class SGDSolverTest : public GradientBasedSolverTest<TypeParam> {
virtual void InitSolver(const SolverParameter& param) {
this->solver_.reset(new SGDSolver<Dtype>(param));
}
-
- virtual SolverParameter_SolverType solver_type() {
- return SolverParameter_SolverType_SGD;
- }
};
TYPED_TEST_CASE(SGDSolverTest, TestDtypesAndDevices);
@@ -721,9 +705,6 @@ class AdaGradSolverTest : public GradientBasedSolverTest<TypeParam> {
virtual void InitSolver(const SolverParameter& param) {
this->solver_.reset(new AdaGradSolver<Dtype>(param));
}
- virtual SolverParameter_SolverType solver_type() {
- return SolverParameter_SolverType_ADAGRAD;
- }
};
TYPED_TEST_CASE(AdaGradSolverTest, TestDtypesAndDevices);
@@ -824,9 +805,6 @@ class NesterovSolverTest : public GradientBasedSolverTest<TypeParam> {
virtual void InitSolver(const SolverParameter& param) {
this->solver_.reset(new NesterovSolver<Dtype>(param));
}
- virtual SolverParameter_SolverType solver_type() {
- return SolverParameter_SolverType_NESTEROV;
- }
};
TYPED_TEST_CASE(NesterovSolverTest, TestDtypesAndDevices);
@@ -960,10 +938,6 @@ class AdaDeltaSolverTest : public GradientBasedSolverTest<TypeParam> {
virtual void InitSolver(const SolverParameter& param) {
this->solver_.reset(new AdaDeltaSolver<Dtype>(param));
}
-
- virtual SolverParameter_SolverType solver_type() {
- return SolverParameter_SolverType_ADADELTA;
- }
};
TYPED_TEST_CASE(AdaDeltaSolverTest, TestDtypesAndDevices);
@@ -1098,9 +1072,6 @@ class AdamSolverTest : public GradientBasedSolverTest<TypeParam> {
new_param.set_momentum2(momentum2);
this->solver_.reset(new AdamSolver<Dtype>(new_param));
}
- virtual SolverParameter_SolverType solver_type() {
- return SolverParameter_SolverType_ADAM;
- }
};
TYPED_TEST_CASE(AdamSolverTest, TestDtypesAndDevices);
@@ -1201,9 +1172,6 @@ class RMSPropSolverTest : public GradientBasedSolverTest<TypeParam> {
new_param.set_rms_decay(rms_decay);
this->solver_.reset(new RMSPropSolver<Dtype>(new_param));
}
- virtual SolverParameter_SolverType solver_type() {
- return SolverParameter_SolverType_RMSPROP;
- }
};
TYPED_TEST_CASE(RMSPropSolverTest, TestDtypesAndDevices);
diff --git a/src/caffe/test/test_hdf5_output_layer.cpp b/src/caffe/test/test_hdf5_output_layer.cpp
index b56277b..3833ebf 100644
--- a/src/caffe/test/test_hdf5_output_layer.cpp
+++ b/src/caffe/test/test_hdf5_output_layer.cpp
@@ -5,10 +5,10 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
+#include "caffe/layers/hdf5_output_layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/hdf5.hpp"
#include "caffe/util/io.hpp"
-#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_hdf5data_layer.cpp b/src/caffe/test/test_hdf5data_layer.cpp
index c9b027f..8884ce9 100644
--- a/src/caffe/test/test_hdf5data_layer.cpp
+++ b/src/caffe/test/test_hdf5data_layer.cpp
@@ -1,13 +1,14 @@
#include <string>
#include <vector>
+#include "hdf5.h"
+
#include "gtest/gtest.h"
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
-#include "caffe/filler.hpp"
+#include "caffe/layers/hdf5_data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
-#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_hinge_loss_layer.cpp b/src/caffe/test/test_hinge_loss_layer.cpp
index b6a9902..8bf89fa 100644
--- a/src/caffe/test/test_hinge_loss_layer.cpp
+++ b/src/caffe/test/test_hinge_loss_layer.cpp
@@ -1,6 +1,4 @@
#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -8,7 +6,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/hinge_loss_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_im2col_kernel.cu b/src/caffe/test/test_im2col_kernel.cu
index f0b75fc..3f97cf6 100644
--- a/src/caffe/test/test_im2col_kernel.cu
+++ b/src/caffe/test/test_im2col_kernel.cu
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,8 +5,8 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/im2col_layer.hpp"
#include "caffe/util/im2col.hpp"
-#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_im2col_layer.cpp b/src/caffe/test/test_im2col_layer.cpp
index 293aa26..8274dd4 100644
--- a/src/caffe/test/test_im2col_layer.cpp
+++ b/src/caffe/test/test_im2col_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/im2col_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_image_data_layer.cpp b/src/caffe/test/test_image_data_layer.cpp
index 481fcef..a4080cc 100644
--- a/src/caffe/test/test_image_data_layer.cpp
+++ b/src/caffe/test/test_image_data_layer.cpp
@@ -8,9 +8,9 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/image_data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/io.hpp"
-#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_infogain_loss_layer.cpp b/src/caffe/test/test_infogain_loss_layer.cpp
index 7ec2f80..a24ac68 100644
--- a/src/caffe/test/test_infogain_loss_layer.cpp
+++ b/src/caffe/test/test_infogain_loss_layer.cpp
@@ -1,6 +1,3 @@
-#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -8,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/loss_layers.hpp"
+#include "caffe/layers/infogain_loss_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_inner_product_layer.cpp b/src/caffe/test/test_inner_product_layer.cpp
index fbf0c85..b888b51 100644
--- a/src/caffe/test/test_inner_product_layer.cpp
+++ b/src/caffe/test/test_inner_product_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/inner_product_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_lrn_layer.cpp b/src/caffe/test/test_lrn_layer.cpp
index c4e2f8e..4c97b1a 100644
--- a/src/caffe/test/test_lrn_layer.cpp
+++ b/src/caffe/test/test_lrn_layer.cpp
@@ -1,5 +1,4 @@
#include <algorithm>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -7,7 +6,12 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/lrn_layer.hpp"
+
+#ifdef USE_CUDNN
+#include "caffe/layers/cudnn_lcn_layer.hpp"
+#include "caffe/layers/cudnn_lrn_layer.hpp"
+#endif
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
@@ -246,5 +250,201 @@ TYPED_TEST(LRNLayerTest, TestGradientWithinChannel) {
this->blob_top_vec_);
}
+#ifdef USE_CUDNN
+template <typename Dtype>
+class CuDNNLRNLayerTest : public GPUDeviceTest<Dtype> {
+ protected:
+ CuDNNLRNLayerTest()
+ : epsilon_(Dtype(1e-5)),
+ blob_bottom_(new Blob<Dtype>()),
+ blob_top_(new Blob<Dtype>()) {}
+ virtual void SetUp() {
+ Caffe::set_random_seed(1701);
+ blob_bottom_->Reshape(2, 7, 3, 3);
+ // fill the values
+ FillerParameter filler_param;
+ GaussianFiller<Dtype> filler(filler_param);
+ filler.Fill(this->blob_bottom_);
+ blob_bottom_vec_.push_back(blob_bottom_);
+ blob_top_vec_.push_back(blob_top_);
+ }
+ virtual ~CuDNNLRNLayerTest() { delete blob_bottom_; delete blob_top_; }
+ void ReferenceLRNForward(const Blob<Dtype>& blob_bottom,
+ const LayerParameter& layer_param, Blob<Dtype>* blob_top);
+
+ Dtype epsilon_;
+ Blob<Dtype>* const blob_bottom_;
+ Blob<Dtype>* const blob_top_;
+ vector<Blob<Dtype>*> blob_bottom_vec_;
+ vector<Blob<Dtype>*> blob_top_vec_;
+};
+
+template <typename TypeParam>
+void CuDNNLRNLayerTest<TypeParam>::ReferenceLRNForward(
+ const Blob<TypeParam>& blob_bottom, const LayerParameter& layer_param,
+ Blob<TypeParam>* blob_top) {
+ typedef TypeParam Dtype;
+ blob_top->Reshape(blob_bottom.num(), blob_bottom.channels(),
+ blob_bottom.height(), blob_bottom.width());
+ Dtype* top_data = blob_top->mutable_cpu_data();
+ LRNParameter lrn_param = layer_param.lrn_param();
+ Dtype alpha = lrn_param.alpha();
+ Dtype beta = lrn_param.beta();
+ int size = lrn_param.local_size();
+ switch (lrn_param.norm_region()) {
+ case LRNParameter_NormRegion_ACROSS_CHANNELS:
+ for (int n = 0; n < blob_bottom.num(); ++n) {
+ for (int c = 0; c < blob_bottom.channels(); ++c) {
+ for (int h = 0; h < blob_bottom.height(); ++h) {
+ for (int w = 0; w < blob_bottom.width(); ++w) {
+ int c_start = c - (size - 1) / 2;
+ int c_end = min(c_start + size, blob_bottom.channels());
+ c_start = max(c_start, 0);
+ Dtype scale = 1.;
+ for (int i = c_start; i < c_end; ++i) {
+ Dtype value = blob_bottom.data_at(n, i, h, w);
+ scale += value * value * alpha / size;
+ }
+ *(top_data + blob_top->offset(n, c, h, w)) =
+ blob_bottom.data_at(n, c, h, w) / pow(scale, beta);
+ }
+ }
+ }
+ }
+ break;
+ case LRNParameter_NormRegion_WITHIN_CHANNEL:
+ for (int n = 0; n < blob_bottom.num(); ++n) {
+ for (int c = 0; c < blob_bottom.channels(); ++c) {
+ for (int h = 0; h < blob_bottom.height(); ++h) {
+ int h_start = h - (size - 1) / 2;
+ int h_end = min(h_start + size, blob_bottom.height());
+ h_start = max(h_start, 0);
+ for (int w = 0; w < blob_bottom.width(); ++w) {
+ Dtype scale = 1.;
+ int w_start = w - (size - 1) / 2;
+ int w_end = min(w_start + size, blob_bottom.width());
+ w_start = max(w_start, 0);
+ for (int nh = h_start; nh < h_end; ++nh) {
+ for (int nw = w_start; nw < w_end; ++nw) {
+ Dtype value = blob_bottom.data_at(n, c, nh, nw);
+ scale += value * value * alpha / (size * size);
+ }
+ }
+ *(top_data + blob_top->offset(n, c, h, w)) =
+ blob_bottom.data_at(n, c, h, w) / pow(scale, beta);
+ }
+ }
+ }
+ }
+ break;
+ default:
+ LOG(FATAL) << "Unknown normalization region.";
+ }
+}
+
+TYPED_TEST_CASE(CuDNNLRNLayerTest, TestDtypes);
+
+TYPED_TEST(CuDNNLRNLayerTest, TestForwardAcrossChannelsCuDNN) {
+ // typedef typename TypeParam::Dtype Dtype;
+ LayerParameter layer_param;
+ CuDNNLRNLayer<TypeParam> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ Blob<TypeParam> top_reference;
+ this->ReferenceLRNForward(*(this->blob_bottom_), layer_param,
+ &top_reference);
+ for (int i = 0; i < this->blob_bottom_->count(); ++i) {
+ EXPECT_NEAR(this->blob_top_->cpu_data()[i], top_reference.cpu_data()[i],
+ this->epsilon_);
+ }
+}
+
+TYPED_TEST(CuDNNLRNLayerTest, TestForwardAcrossChannelsLargeRegionCuDNN) {
+ typedef TypeParam Dtype;
+ LayerParameter layer_param;
+ layer_param.mutable_lrn_param()->set_local_size(15);
+ CuDNNLRNLayer<Dtype> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ Blob<Dtype> top_reference;
+ this->ReferenceLRNForward(*(this->blob_bottom_), layer_param,
+ &top_reference);
+ for (int i = 0; i < this->blob_bottom_->count(); ++i) {
+ EXPECT_NEAR(this->blob_top_->cpu_data()[i], top_reference.cpu_data()[i],
+ this->epsilon_);
+ }
+}
+
+TYPED_TEST(CuDNNLRNLayerTest, TestGradientAcrossChannelsCuDNN) {
+ typedef TypeParam Dtype;
+ LayerParameter layer_param;
+ CuDNNLRNLayer<Dtype> layer(layer_param);
+ GradientChecker<Dtype> checker(1e-2, 1e-2);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ for (int i = 0; i < this->blob_top_->count(); ++i) {
+ this->blob_top_->mutable_cpu_diff()[i] = 1.;
+ }
+ vector<bool> propagate_down(this->blob_bottom_vec_.size(), true);
+ layer.Backward(this->blob_top_vec_, propagate_down,
+ this->blob_bottom_vec_);
+ checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_,
+ this->blob_top_vec_);
+}
+
+TYPED_TEST(CuDNNLRNLayerTest, TestForwardWithinChannel) {
+ typedef TypeParam Dtype;
+ LayerParameter layer_param;
+ layer_param.mutable_lrn_param()->set_norm_region(
+ LRNParameter_NormRegion_WITHIN_CHANNEL);
+ layer_param.mutable_lrn_param()->set_local_size(3);
+ CuDNNLCNLayer<Dtype> layer(layer_param);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ Blob<Dtype> top_reference;
+ this->ReferenceLRNForward(*(this->blob_bottom_), layer_param,
+ &top_reference);
+ for (int i = 0; i < this->blob_bottom_->count(); ++i) {
+ EXPECT_NEAR(this->blob_top_->cpu_data()[i], top_reference.cpu_data()[i],
+ this->epsilon_);
+ }
+}
+
+TYPED_TEST(CuDNNLRNLayerTest, TestGradientWithinChannel) {
+ typedef TypeParam Dtype;
+ LayerParameter layer_param;
+ layer_param.mutable_lrn_param()->set_norm_region(
+ LRNParameter_NormRegion_WITHIN_CHANNEL);
+ layer_param.mutable_lrn_param()->set_local_size(3);
+ CuDNNLCNLayer<Dtype> layer(layer_param);
+ GradientChecker<Dtype> checker(1e-2, 1e-2);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ for (int i = 0; i < this->blob_top_->count(); ++i) {
+ this->blob_top_->mutable_cpu_diff()[i] = 1.;
+ }
+ checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_,
+ this->blob_top_vec_);
+}
+
+TYPED_TEST(CuDNNLRNLayerTest, TestGradientAcrossChannelsLargeRegionCuDNN) {
+ typedef TypeParam Dtype;
+ LayerParameter layer_param;
+ layer_param.mutable_lrn_param()->set_local_size(15);
+ CuDNNLRNLayer<Dtype> layer(layer_param);
+ GradientChecker<Dtype> checker(1e-2, 1e-2);
+ layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+ layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+ for (int i = 0; i < this->blob_top_->count(); ++i) {
+ this->blob_top_->mutable_cpu_diff()[i] = 1.;
+ }
+ vector<bool> propagate_down(this->blob_bottom_vec_.size(), true);
+ layer.Backward(this->blob_top_vec_, propagate_down,
+ this->blob_bottom_vec_);
+ checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_,
+ this->blob_top_vec_);
+}
+
+#endif
} // namespace caffe
diff --git a/src/caffe/test/test_math_functions.cpp b/src/caffe/test/test_math_functions.cpp
index a095b54..efc5a27 100644
--- a/src/caffe/test/test_math_functions.cpp
+++ b/src/caffe/test/test_math_functions.cpp
@@ -1,8 +1,6 @@
#include <stdint.h> // for uint32_t & uint64_t
#include <time.h>
-#include <climits>
#include <cmath> // for std::fabs
-#include <cstdlib> // for rand_r
#include "gtest/gtest.h"
@@ -41,27 +39,6 @@ class MathFunctionsTest : public MultiDeviceTest<TypeParam> {
delete blob_top_;
}
- // http://en.wikipedia.org/wiki/Hamming_distance
- int ReferenceHammingDistance(const int n, const Dtype* x, const Dtype* y) {
- int dist = 0;
- uint64_t val;
- for (int i = 0; i < n; ++i) {
- if (sizeof(Dtype) == 8) {
- val = static_cast<uint64_t>(x[i]) ^ static_cast<uint64_t>(y[i]);
- } else if (sizeof(Dtype) == 4) {
- val = static_cast<uint32_t>(x[i]) ^ static_cast<uint32_t>(y[i]);
- } else {
- LOG(FATAL) << "Unrecognized Dtype size: " << sizeof(Dtype);
- }
- // Count the number of set bits
- while (val) {
- ++dist;
- val &= val - 1;
- }
- }
- return dist;
- }
-
Blob<Dtype>* const blob_bottom_;
Blob<Dtype>* const blob_top_;
};
@@ -78,14 +55,6 @@ TYPED_TEST(CPUMathFunctionsTest, TestNothing) {
// due to the set up overhead.
}
-TYPED_TEST(CPUMathFunctionsTest, TestHammingDistance) {
- int n = this->blob_bottom_->count();
- const TypeParam* x = this->blob_bottom_->cpu_data();
- const TypeParam* y = this->blob_top_->cpu_data();
- EXPECT_EQ(this->ReferenceHammingDistance(n, x, y),
- caffe_cpu_hamming_distance<TypeParam>(n, x, y));
-}
-
TYPED_TEST(CPUMathFunctionsTest, TestAsum) {
int n = this->blob_bottom_->count();
const TypeParam* x = this->blob_bottom_->cpu_data();
@@ -158,18 +127,6 @@ class GPUMathFunctionsTest : public MathFunctionsTest<GPUDevice<Dtype> > {
TYPED_TEST_CASE(GPUMathFunctionsTest, TestDtypes);
-// TODO: Fix caffe_gpu_hamming_distance and re-enable this test.
-TYPED_TEST(GPUMathFunctionsTest, DISABLED_TestHammingDistance) {
- int n = this->blob_bottom_->count();
- const TypeParam* x = this->blob_bottom_->cpu_data();
- const TypeParam* y = this->blob_top_->cpu_data();
- int reference_distance = this->ReferenceHammingDistance(n, x, y);
- x = this->blob_bottom_->gpu_data();
- y = this->blob_top_->gpu_data();
- int computed_distance = caffe_gpu_hamming_distance<TypeParam>(n, x, y);
- EXPECT_EQ(reference_distance, computed_distance);
-}
-
TYPED_TEST(GPUMathFunctionsTest, TestAsum) {
int n = this->blob_bottom_->count();
const TypeParam* x = this->blob_bottom_->cpu_data();
diff --git a/src/caffe/test/test_maxpool_dropout_layers.cpp b/src/caffe/test/test_maxpool_dropout_layers.cpp
index 611d979..4f0e20a 100644
--- a/src/caffe/test/test_maxpool_dropout_layers.cpp
+++ b/src/caffe/test/test_maxpool_dropout_layers.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,8 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/dropout_layer.hpp"
+#include "caffe/layers/pooling_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_memory_data_layer.cpp b/src/caffe/test/test_memory_data_layer.cpp
index 7269a4d..7998bc1 100644
--- a/src/caffe/test/test_memory_data_layer.cpp
+++ b/src/caffe/test/test_memory_data_layer.cpp
@@ -5,8 +5,8 @@
#include <string>
#include <vector>
-#include "caffe/data_layers.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/memory_data_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_multinomial_logistic_loss_layer.cpp b/src/caffe/test/test_multinomial_logistic_loss_layer.cpp
index b2db984..8cc2102 100644
--- a/src/caffe/test/test_multinomial_logistic_loss_layer.cpp
+++ b/src/caffe/test/test_multinomial_logistic_loss_layer.cpp
@@ -1,6 +1,3 @@
-#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -8,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/multinomial_logistic_loss_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_mvn_layer.cpp b/src/caffe/test/test_mvn_layer.cpp
index be23d86..28a762d 100644
--- a/src/caffe/test/test_mvn_layer.cpp
+++ b/src/caffe/test/test_mvn_layer.cpp
@@ -1,11 +1,9 @@
-#include <cmath>
-#include <cstring>
#include <vector>
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
-#include "caffe/common_layers.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/mvn_layer.hpp"
#include "google/protobuf/text_format.h"
#include "gtest/gtest.h"
diff --git a/src/caffe/test/test_neuron_layer.cpp b/src/caffe/test/test_neuron_layer.cpp
index c6e4d27..21441b4 100644
--- a/src/caffe/test/test_neuron_layer.cpp
+++ b/src/caffe/test/test_neuron_layer.cpp
@@ -1,5 +1,4 @@
#include <algorithm>
-#include <cstring>
#include <vector>
#include "google/protobuf/text_format.h"
@@ -8,7 +7,25 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+
+#include "caffe/layers/absval_layer.hpp"
+#include "caffe/layers/bnll_layer.hpp"
+#include "caffe/layers/dropout_layer.hpp"
+#include "caffe/layers/exp_layer.hpp"
+#include "caffe/layers/inner_product_layer.hpp"
+#include "caffe/layers/log_layer.hpp"
+#include "caffe/layers/power_layer.hpp"
+#include "caffe/layers/prelu_layer.hpp"
+#include "caffe/layers/relu_layer.hpp"
+#include "caffe/layers/sigmoid_layer.hpp"
+#include "caffe/layers/tanh_layer.hpp"
+#include "caffe/layers/threshold_layer.hpp"
+
+#ifdef USE_CUDNN
+#include "caffe/layers/cudnn_relu_layer.hpp"
+#include "caffe/layers/cudnn_sigmoid_layer.hpp"
+#include "caffe/layers/cudnn_tanh_layer.hpp"
+#endif
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_pooling_layer.cpp b/src/caffe/test/test_pooling_layer.cpp
index 69f2d5c..bb95cae 100644
--- a/src/caffe/test/test_pooling_layer.cpp
+++ b/src/caffe/test/test_pooling_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,11 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/pooling_layer.hpp"
+
+#ifdef USE_CUDNN
+#include "caffe/layers/cudnn_pooling_layer.hpp"
+#endif
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_power_layer.cpp b/src/caffe/test/test_power_layer.cpp
index 76c9e85..1aa587a 100644
--- a/src/caffe/test/test_power_layer.cpp
+++ b/src/caffe/test/test_power_layer.cpp
@@ -6,7 +6,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/power_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_random_number_generator.cpp b/src/caffe/test/test_random_number_generator.cpp
index 98424c0..833b004 100644
--- a/src/caffe/test/test_random_number_generator.cpp
+++ b/src/caffe/test/test_random_number_generator.cpp
@@ -1,5 +1,4 @@
#include <cmath>
-#include <cstring>
#include "gtest/gtest.h"
diff --git a/src/caffe/test/test_reduction_layer.cpp b/src/caffe/test/test_reduction_layer.cpp
index f568a18..6ed7cda 100644
--- a/src/caffe/test/test_reduction_layer.cpp
+++ b/src/caffe/test/test_reduction_layer.cpp
@@ -1,4 +1,3 @@
-#include <algorithm>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/reduction_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_reshape_layer.cpp b/src/caffe/test/test_reshape_layer.cpp
index 9d08ec6..4f26138 100644
--- a/src/caffe/test/test_reshape_layer.cpp
+++ b/src/caffe/test/test_reshape_layer.cpp
@@ -1,12 +1,11 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
-#include "caffe/common_layers.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/reshape_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_sigmoid_cross_entropy_loss_layer.cpp b/src/caffe/test/test_sigmoid_cross_entropy_loss_layer.cpp
index e5737e4..5dfd765 100644
--- a/src/caffe/test/test_sigmoid_cross_entropy_loss_layer.cpp
+++ b/src/caffe/test/test_sigmoid_cross_entropy_loss_layer.cpp
@@ -1,6 +1,4 @@
#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -8,7 +6,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/sigmoid_cross_entropy_loss_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_slice_layer.cpp b/src/caffe/test/test_slice_layer.cpp
index 2d2d0fd..c2b231e 100644
--- a/src/caffe/test/test_slice_layer.cpp
+++ b/src/caffe/test/test_slice_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/slice_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_softmax_layer.cpp b/src/caffe/test/test_softmax_layer.cpp
index 996da4b..9444357 100644
--- a/src/caffe/test/test_softmax_layer.cpp
+++ b/src/caffe/test/test_softmax_layer.cpp
@@ -1,5 +1,4 @@
#include <cmath>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -7,7 +6,11 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/softmax_layer.hpp"
+
+#ifdef USE_CUDNN
+#include "caffe/layers/cudnn_softmax_layer.hpp"
+#endif
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_softmax_with_loss_layer.cpp b/src/caffe/test/test_softmax_with_loss_layer.cpp
index 1498d5c..c67f3e0 100644
--- a/src/caffe/test/test_softmax_with_loss_layer.cpp
+++ b/src/caffe/test/test_softmax_with_loss_layer.cpp
@@ -1,6 +1,4 @@
#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "boost/scoped_ptr.hpp"
@@ -9,7 +7,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/softmax_loss_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_solver.cpp b/src/caffe/test/test_solver.cpp
index ceabc9c..b181642 100644
--- a/src/caffe/test/test_solver.cpp
+++ b/src/caffe/test/test_solver.cpp
@@ -7,6 +7,7 @@
#include "caffe/common.hpp"
#include "caffe/proto/caffe.pb.h"
+#include "caffe/sgd_solvers.hpp"
#include "caffe/solver.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_solver_factory.cpp b/src/caffe/test/test_solver_factory.cpp
new file mode 100644
index 0000000..eef5290
--- /dev/null
+++ b/src/caffe/test/test_solver_factory.cpp
@@ -0,0 +1,50 @@
+#include <map>
+#include <string>
+
+#include "boost/scoped_ptr.hpp"
+#include "google/protobuf/text_format.h"
+#include "gtest/gtest.h"
+
+#include "caffe/common.hpp"
+#include "caffe/solver.hpp"
+#include "caffe/solver_factory.hpp"
+
+#include "caffe/test/test_caffe_main.hpp"
+
+namespace caffe {
+
+template <typename TypeParam>
+class SolverFactoryTest : public MultiDeviceTest<TypeParam> {
+ protected:
+ SolverParameter simple_solver_param() {
+ const string solver_proto =
+ "train_net_param { "
+ " layer { "
+ " name: 'data' type: 'DummyData' top: 'data' "
+ " dummy_data_param { shape { dim: 1 } } "
+ " } "
+ "} ";
+ SolverParameter solver_param;
+ CHECK(google::protobuf::TextFormat::ParseFromString(
+ solver_proto, &solver_param));
+ return solver_param;
+ }
+};
+
+TYPED_TEST_CASE(SolverFactoryTest, TestDtypesAndDevices);
+
+TYPED_TEST(SolverFactoryTest, TestCreateSolver) {
+ typedef typename TypeParam::Dtype Dtype;
+ typename SolverRegistry<Dtype>::CreatorRegistry& registry =
+ SolverRegistry<Dtype>::Registry();
+ shared_ptr<Solver<Dtype> > solver;
+ SolverParameter solver_param = this->simple_solver_param();
+ for (typename SolverRegistry<Dtype>::CreatorRegistry::iterator iter =
+ registry.begin(); iter != registry.end(); ++iter) {
+ solver_param.set_type(iter->first);
+ solver.reset(SolverRegistry<Dtype>::CreateSolver(solver_param));
+ EXPECT_EQ(iter->first, solver->type());
+ }
+}
+
+} // namespace caffe
diff --git a/src/caffe/test/test_split_layer.cpp b/src/caffe/test/test_split_layer.cpp
index be5204b..ba2ccbb 100644
--- a/src/caffe/test/test_split_layer.cpp
+++ b/src/caffe/test/test_split_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <string>
#include <vector>
@@ -8,9 +7,9 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/split_layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/insert_splits.hpp"
-#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_spp_layer.cpp b/src/caffe/test/test_spp_layer.cpp
index b2585f1..59a3af2 100644
--- a/src/caffe/test/test_spp_layer.cpp
+++ b/src/caffe/test/test_spp_layer.cpp
@@ -1,5 +1,3 @@
-#include <algorithm>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -7,7 +5,12 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/concat_layer.hpp"
+#include "caffe/layers/flatten_layer.hpp"
+#include "caffe/layers/pooling_layer.hpp"
+#include "caffe/layers/split_layer.hpp"
+#include "caffe/layers/spp_layer.hpp"
+
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_stochastic_pooling.cpp b/src/caffe/test/test_stochastic_pooling.cpp
index f84464c..cd5db83 100644
--- a/src/caffe/test/test_stochastic_pooling.cpp
+++ b/src/caffe/test/test_stochastic_pooling.cpp
@@ -1,5 +1,4 @@
#include <algorithm>
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -7,7 +6,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/pooling_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_syncedmem.cpp b/src/caffe/test/test_syncedmem.cpp
index b946233..16dfb58 100644
--- a/src/caffe/test/test_syncedmem.cpp
+++ b/src/caffe/test/test_syncedmem.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
diff --git a/src/caffe/test/test_tanh_layer.cpp b/src/caffe/test/test_tanh_layer.cpp
index 5dc9283..bb8699a 100644
--- a/src/caffe/test/test_tanh_layer.cpp
+++ b/src/caffe/test/test_tanh_layer.cpp
@@ -5,8 +5,8 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
-#include "caffe/common_layers.hpp"
#include "caffe/filler.hpp"
+#include "caffe/layers/tanh_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_threshold_layer.cpp b/src/caffe/test/test_threshold_layer.cpp
index 05ce821..1e84cc5 100644
--- a/src/caffe/test/test_threshold_layer.cpp
+++ b/src/caffe/test/test_threshold_layer.cpp
@@ -5,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/threshold_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
diff --git a/src/caffe/test/test_tile_layer.cpp b/src/caffe/test/test_tile_layer.cpp
index 540aac3..7ff7552 100644
--- a/src/caffe/test/test_tile_layer.cpp
+++ b/src/caffe/test/test_tile_layer.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <vector>
#include "gtest/gtest.h"
@@ -6,7 +5,7 @@
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
-#include "caffe/vision_layers.hpp"
+#include "caffe/layers/tile_layer.hpp"
#include "caffe/test/test_caffe_main.hpp"
#include "caffe/test/test_gradient_check_util.hpp"
diff --git a/src/caffe/test/test_upgrade_proto.cpp b/src/caffe/test/test_upgrade_proto.cpp
index ee05b15..9dcc2aa 100644
--- a/src/caffe/test/test_upgrade_proto.cpp
+++ b/src/caffe/test/test_upgrade_proto.cpp
@@ -1,4 +1,3 @@
-#include <cstring>
#include <string>
#include <vector>
@@ -2892,7 +2891,6 @@ TEST_F(NetUpgradeTest, TestImageNet) {
this->RunV1UpgradeTest(expected_v1_proto, expected_v2_proto);
} // NOLINT(readability/fn_size)
-#ifdef USE_OPENCV
TEST_F(NetUpgradeTest, TestUpgradeV1LayerType) {
LayerParameter layer_param;
shared_ptr<Layer<float> > layer;
@@ -2927,5 +2925,65 @@ TEST_F(NetUpgradeTest, TestUpgradeV1LayerType) {
EXPECT_EQ(v2_layer_type, layer->type());
}
}
-#endif // USE_OPENCV
+
+class SolverTypeUpgradeTest : public ::testing::Test {
+ protected:
+ void RunSolverTypeUpgradeTest(
+ const string& input_param_string, const string& output_param_string) {
+ // Test upgrading old solver_type field (enum) to new type field (string)
+ SolverParameter input_param;
+ CHECK(google::protobuf::TextFormat::ParseFromString(
+ input_param_string, &input_param));
+ SolverParameter expected_output_param;
+ CHECK(google::protobuf::TextFormat::ParseFromString(
+ output_param_string, &expected_output_param));
+ SolverParameter actual_output_param = input_param;
+ UpgradeSolverType(&actual_output_param);
+ EXPECT_EQ(expected_output_param.DebugString(),
+ actual_output_param.DebugString());
+ }
+};
+
+TEST_F(SolverTypeUpgradeTest, TestSimple) {
+ const char* old_type_vec[6] = { "SGD", "ADAGRAD", "NESTEROV", "RMSPROP",
+ "ADADELTA", "ADAM" };
+ const char* new_type_vec[6] = { "SGD", "AdaGrad", "Nesterov", "RMSProp",
+ "AdaDelta", "Adam" };
+ for (int i = 0; i < 6; ++i) {
+ const string& input_proto =
+ "net: 'examples/mnist/lenet_train_test.prototxt' "
+ "test_iter: 100 "
+ "test_interval: 500 "
+ "base_lr: 0.01 "
+ "momentum: 0.0 "
+ "weight_decay: 0.0005 "
+ "lr_policy: 'inv' "
+ "gamma: 0.0001 "
+ "power: 0.75 "
+ "display: 100 "
+ "max_iter: 10000 "
+ "snapshot: 5000 "
+ "snapshot_prefix: 'examples/mnist/lenet_rmsprop' "
+ "solver_mode: GPU "
+ "solver_type: " + std::string(old_type_vec[i]) + " ";
+ const string& expected_output_proto =
+ "net: 'examples/mnist/lenet_train_test.prototxt' "
+ "test_iter: 100 "
+ "test_interval: 500 "
+ "base_lr: 0.01 "
+ "momentum: 0.0 "
+ "weight_decay: 0.0005 "
+ "lr_policy: 'inv' "
+ "gamma: 0.0001 "
+ "power: 0.75 "
+ "display: 100 "
+ "max_iter: 10000 "
+ "snapshot: 5000 "
+ "snapshot_prefix: 'examples/mnist/lenet_rmsprop' "
+ "solver_mode: GPU "
+ "type: '" + std::string(new_type_vec[i]) + "' ";
+ this->RunSolverTypeUpgradeTest(input_proto, expected_output_proto);
+ }
+}
+
} // NOLINT(readability/fn_size) // namespace caffe
diff --git a/src/caffe/test/test_util_blas.cpp b/src/caffe/test/test_util_blas.cpp
index 8770f30..9ee8818 100644
--- a/src/caffe/test/test_util_blas.cpp
+++ b/src/caffe/test/test_util_blas.cpp
@@ -1,7 +1,5 @@
#ifndef CPU_ONLY // CPU-GPU test
-#include <cstring>
-
#include "gtest/gtest.h"
#include "caffe/blob.hpp"
diff --git a/src/caffe/util/blocking_queue.cpp b/src/caffe/util/blocking_queue.cpp
index d1d1fa8..058668f 100644
--- a/src/caffe/util/blocking_queue.cpp
+++ b/src/caffe/util/blocking_queue.cpp
@@ -1,8 +1,8 @@
#include <boost/thread.hpp>
#include <string>
-#include "caffe/data_layers.hpp"
#include "caffe/data_reader.hpp"
+#include "caffe/layers/base_data_layer.hpp"
#include "caffe/parallel.hpp"
#include "caffe/util/blocking_queue.hpp"
diff --git a/src/caffe/util/db.cpp b/src/caffe/util/db.cpp
index ccda054..7f22509 100644
--- a/src/caffe/util/db.cpp
+++ b/src/caffe/util/db.cpp
@@ -18,6 +18,7 @@ DB* GetDB(DataParameter::DB backend) {
#endif // USE_LMDB
default:
LOG(FATAL) << "Unknown database backend";
+ return NULL;
}
}
@@ -33,6 +34,7 @@ DB* GetDB(const string& backend) {
}
#endif // USE_LMDB
LOG(FATAL) << "Unknown database backend";
+ return NULL;
}
} // namespace db
diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp
index 78dd880..0bc82b5 100644
--- a/src/caffe/util/db_lmdb.cpp
+++ b/src/caffe/util/db_lmdb.cpp
@@ -19,7 +19,22 @@ void LMDB::Open(const string& source, Mode mode) {
if (mode == READ) {
flags = MDB_RDONLY | MDB_NOTLS;
}
- MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664));
+ int rc = mdb_env_open(mdb_env_, source.c_str(), flags, 0664);
+#ifndef ALLOW_LMDB_NOLOCK
+ MDB_CHECK(rc);
+#else
+ if (rc == EACCES) {
+ LOG(WARNING) << "Permission denied. Trying with MDB_NOLOCK ...";
+ // Close and re-open environment handle
+ mdb_env_close(mdb_env_);
+ MDB_CHECK(mdb_env_create(&mdb_env_));
+ // Try again with MDB_NOLOCK
+ flags |= MDB_NOLOCK;
+ MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664));
+ } else {
+ MDB_CHECK(rc);
+ }
+#endif
LOG(INFO) << "Opened lmdb " << source;
}
diff --git a/src/caffe/util/im2col.cpp b/src/caffe/util/im2col.cpp
index b0a7be5..27e5b7c 100644
--- a/src/caffe/util/im2col.cpp
+++ b/src/caffe/util/im2col.cpp
@@ -1,6 +1,3 @@
-#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include <vector>
#include "caffe/util/im2col.hpp"
@@ -14,22 +11,20 @@ void im2col_cpu(const Dtype* data_im, const int channels,
const int pad_h, const int pad_w,
const int stride_h, const int stride_w,
Dtype* data_col) {
- int height_col = (height + 2 * pad_h - kernel_h) / stride_h + 1;
- int width_col = (width + 2 * pad_w - kernel_w) / stride_w + 1;
- int channels_col = channels * kernel_h * kernel_w;
- for (int c = 0; c < channels_col; ++c) {
- int w_offset = c % kernel_w;
- int h_offset = (c / kernel_w) % kernel_h;
- int c_im = c / kernel_h / kernel_w;
- for (int h = 0; h < height_col; ++h) {
- for (int w = 0; w < width_col; ++w) {
- int h_pad = h * stride_h - pad_h + h_offset;
- int w_pad = w * stride_w - pad_w + w_offset;
- if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width)
- data_col[(c * height_col + h) * width_col + w] =
- data_im[(c_im * height + h_pad) * width + w_pad];
- else
- data_col[(c * height_col + h) * width_col + w] = 0;
+ const int height_col = (height + 2 * pad_h - kernel_h) / stride_h + 1;
+ const int width_col = (width + 2 * pad_w - kernel_w) / stride_w + 1;
+ const int channels_col = channels * kernel_h * kernel_w;
+ for (int c_col = 0; c_col < channels_col; ++c_col) {
+ int w_offset = c_col % kernel_w;
+ int h_offset = (c_col / kernel_w) % kernel_h;
+ int c_im = c_col / kernel_h / kernel_w;
+ for (int h_col = 0; h_col < height_col; ++h_col) {
+ for (int w_col = 0; w_col < width_col; ++w_col) {
+ int h_im = h_col * stride_h - pad_h + h_offset;
+ int w_im = w_col * stride_w - pad_w + w_offset;
+ data_col[(c_col * height_col + h_col) * width_col + w_col] =
+ (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) ?
+ data_im[(c_im * height + h_im) * width + w_im] : 0;
}
}
}
@@ -64,9 +59,9 @@ inline void im2col_nd_core_cpu(const Dtype* data_input, const bool im2col,
const int channels_col = col_shape[0];
vector<int> d_offset(num_spatial_axes, 0);
vector<int> d_iter(num_spatial_axes, 0);
- for (int c = 0; c < channels_col; ++c) {
+ for (int c_col = 0; c_col < channels_col; ++c_col) {
// Loop over spatial axes in reverse order to compute a per-axis offset.
- int offset = c;
+ int offset = c_col;
for (int d_i = num_spatial_axes - 1; d_i >= 0; --d_i) {
if (d_i < num_spatial_axes - 1) {
offset /= kernel_shape[d_i + 1];
@@ -76,17 +71,17 @@ inline void im2col_nd_core_cpu(const Dtype* data_input, const bool im2col,
for (bool incremented = true; incremented; ) {
// Loop over spatial axes in forward order to compute the indices in the
// image and column, and whether the index lies in the padding.
- int index_col = c;
- int index_im = c / kernel_size;
+ int index_col = c_col;
+ int index_im = c_col / kernel_size;
bool is_padding = false;
for (int d_i = 0; d_i < num_spatial_axes; ++d_i) {
const int d = d_iter[d_i];
- const int d_pad = d * stride[d_i] - pad[d_i] + d_offset[d_i];
- is_padding |= d_pad < 0 || d_pad >= im_shape[d_i + 1];
+ const int d_im = d * stride[d_i] - pad[d_i] + d_offset[d_i];
+ is_padding |= d_im < 0 || d_im >= im_shape[d_i + 1];
index_col *= col_shape[d_i + 1];
index_col += d;
index_im *= im_shape[d_i + 1];
- index_im += d_pad;
+ index_im += d_im;
}
if (im2col) {
if (is_padding) {
@@ -139,25 +134,25 @@ template void im2col_nd_cpu<double>(const double* data_im,
template <typename Dtype>
void col2im_cpu(const Dtype* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w,
const int stride_h, const int stride_w,
Dtype* data_im) {
caffe_set(height * width * channels, Dtype(0), data_im);
- int height_col = (height + 2 * pad_h - patch_h) / stride_h + 1;
- int width_col = (width + 2 * pad_w - patch_w) / stride_w + 1;
- int channels_col = channels * patch_h * patch_w;
- for (int c = 0; c < channels_col; ++c) {
- int w_offset = c % patch_w;
- int h_offset = (c / patch_w) % patch_h;
- int c_im = c / patch_h / patch_w;
- for (int h = 0; h < height_col; ++h) {
- for (int w = 0; w < width_col; ++w) {
- int h_pad = h * stride_h - pad_h + h_offset;
- int w_pad = w * stride_w - pad_w + w_offset;
- if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width)
- data_im[(c_im * height + h_pad) * width + w_pad] +=
- data_col[(c * height_col + h) * width_col + w];
+ const int height_col = (height + 2 * pad_h - kernel_h) / stride_h + 1;
+ const int width_col = (width + 2 * pad_w - kernel_w) / stride_w + 1;
+ const int channels_col = channels * kernel_h * kernel_w;
+ for (int c_col = 0; c_col < channels_col; ++c_col) {
+ int w_offset = c_col % kernel_w;
+ int h_offset = (c_col / kernel_w) % kernel_h;
+ int c_im = c_col / kernel_h / kernel_w;
+ for (int h_col = 0; h_col < height_col; ++h_col) {
+ for (int w_col = 0; w_col < width_col; ++w_col) {
+ int h_im = h_col * stride_h - pad_h + h_offset;
+ int w_im = w_col * stride_w - pad_w + w_offset;
+ if (h_im >= 0 && h_im < height && w_im >= 0 && w_im < width)
+ data_im[(c_im * height + h_im) * width + w_im] +=
+ data_col[(c_col * height_col + h_col) * width_col + w_col];
}
}
}
@@ -165,11 +160,11 @@ void col2im_cpu(const Dtype* data_col, const int channels,
// Explicit instantiation
template void col2im_cpu<float>(const float* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w, const int stride_h,
const int stride_w, float* data_im);
template void col2im_cpu<double>(const double* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w, const int stride_h,
const int stride_w, double* data_im);
diff --git a/src/caffe/util/im2col.cu b/src/caffe/util/im2col.cu
index 5a478ba..49354ab 100644
--- a/src/caffe/util/im2col.cu
+++ b/src/caffe/util/im2col.cu
@@ -1,7 +1,4 @@
#include <algorithm>
-#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include "caffe/common.hpp"
#include "caffe/util/im2col.hpp"
@@ -16,22 +13,23 @@ __global__ void im2col_gpu_kernel(const int n, const Dtype* data_im,
const int height_col, const int width_col,
Dtype* data_col) {
CUDA_KERNEL_LOOP(index, n) {
- int w_out = index % width_col;
- int h_index = index / width_col;
- int h_out = h_index % height_col;
- int channel_in = h_index / height_col;
- int channel_out = channel_in * kernel_h * kernel_w;
- int h_in = h_out * stride_h - pad_h;
- int w_in = w_out * stride_w - pad_w;
+ const int h_index = index / width_col;
+ const int h_col = h_index % height_col;
+ const int w_col = index % width_col;
+ const int c_im = h_index / height_col;
+ const int c_col = c_im * kernel_h * kernel_w;
+ const int h_offset = h_col * stride_h - pad_h;
+ const int w_offset = w_col * stride_w - pad_w;
Dtype* data_col_ptr = data_col;
- data_col_ptr += (channel_out * height_col + h_out) * width_col + w_out;
+ data_col_ptr += (c_col * height_col + h_col) * width_col + w_col;
const Dtype* data_im_ptr = data_im;
- data_im_ptr += (channel_in * height + h_in) * width + w_in;
+ data_im_ptr += (c_im * height + h_offset) * width + w_offset;
for (int i = 0; i < kernel_h; ++i) {
for (int j = 0; j < kernel_w; ++j) {
- int h = h_in + i;
- int w = w_in + j;
- *data_col_ptr = (h >= 0 && w >= 0 && h < height && w < width) ?
+ int h_im = h_offset + i;
+ int w_im = w_offset + j;
+ *data_col_ptr =
+ (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) ?
data_im_ptr[i * width + j] : 0;
data_col_ptr += height_col * width_col;
}
@@ -222,35 +220,39 @@ template void im2col_nd_gpu<double>(const double* data_im,
template <typename Dtype>
__global__ void col2im_gpu_kernel(const int n, const Dtype* data_col,
const int height, const int width, const int channels,
- const int patch_h, const int patch_w,
+ const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w,
const int stride_h, const int stride_w,
const int height_col, const int width_col,
Dtype* data_im) {
CUDA_KERNEL_LOOP(index, n) {
Dtype val = 0;
- int w = index % width + pad_w;
- int h = (index / width) % height + pad_h;
- int c = index / (width * height);
+ const int w_im = index % width + pad_w;
+ const int h_im = (index / width) % height + pad_h;
+ const int c_im = index / (width * height);
// compute the start and end of the output
- int w_col_start = (w < patch_w) ? 0 : (w - patch_w) / stride_w + 1;
- int w_col_end = min(w / stride_w + 1, width_col);
- int h_col_start = (h < patch_h) ? 0 : (h - patch_h) / stride_h + 1;
- int h_col_end = min(h / stride_h + 1, height_col);
+ const int w_col_start =
+ (w_im < kernel_w) ? 0 : (w_im - kernel_w) / stride_w + 1;
+ const int w_col_end =
+ min(w_im / stride_w + 1, width_col);
+ const int h_col_start =
+ (h_im < kernel_h) ? 0 : (h_im - kernel_h) / stride_h + 1;
+ const int h_col_end =
+ min(h_im / stride_h + 1, height_col);
/*
for (int h_col = h_col_start; h_col < h_col_end; ++h_col) {
for (int w_col = w_col_start; w_col < w_col_end; ++w_col) {
// the col location: [c * width * height + h_out, w_out]
- int c_col = c * patch_h * patch_w + (h - h_col * stride_h) * ksize
- + (w - w_col * stride_w);
+ int c_col = c_im * kernel_h * kernel_w
+ + (h_im - h_col * stride_h) * kernel_w + (w_im - w_col * stride_w);
val += data_col[(c_col * height_col + h_col) * width_col + w_col];
}
}
*/
// equivalent implementation
- int offset =
- (c * patch_h * patch_w + h * patch_w + w) * height_col * width_col;
- int coeff_h_col = (1 - stride_h * patch_w * height_col) * width_col;
+ int offset = (c_im * kernel_h * kernel_w + h_im * kernel_w + w_im)
+ * height_col * width_col;
+ int coeff_h_col = (1 - stride_h * kernel_w * height_col) * width_col;
int coeff_w_col = (1 - stride_w * height_col * width_col);
for (int h_col = h_col_start; h_col < h_col_end; ++h_col) {
for (int w_col = w_col_start; w_col < w_col_end; ++w_col) {
@@ -263,18 +265,18 @@ __global__ void col2im_gpu_kernel(const int n, const Dtype* data_col,
template <typename Dtype>
void col2im_gpu(const Dtype* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w, const int stride_h,
const int stride_w, Dtype* data_im) {
- int height_col = (height + 2 * pad_h - patch_h) / stride_h + 1;
- int width_col = (width + 2 * pad_w - patch_w) / stride_w + 1;
+ int height_col = (height + 2 * pad_h - kernel_h) / stride_h + 1;
+ int width_col = (width + 2 * pad_w - kernel_w) / stride_w + 1;
int num_kernels = channels * height * width;
// To avoid involving atomic operations, we will launch one kernel per
// bottom dimension, and then in the kernel add up the top dimensions.
// NOLINT_NEXT_LINE(whitespace/operators)
col2im_gpu_kernel<Dtype><<<CAFFE_GET_BLOCKS(num_kernels),
CAFFE_CUDA_NUM_THREADS>>>(
- num_kernels, data_col, height, width, channels, patch_h, patch_w,
+ num_kernels, data_col, height, width, channels, kernel_h, kernel_w,
pad_h, pad_w, stride_h, stride_w,
height_col, width_col, data_im);
CUDA_POST_KERNEL_CHECK;
@@ -282,11 +284,11 @@ void col2im_gpu(const Dtype* data_col, const int channels,
// Explicit instantiation
template void col2im_gpu<float>(const float* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w, const int stride_h,
const int stride_w, float* data_im);
template void col2im_gpu<double>(const double* data_col, const int channels,
- const int height, const int width, const int patch_h, const int patch_w,
+ const int height, const int width, const int kernel_h, const int kernel_w,
const int pad_h, const int pad_w, const int stride_h,
const int stride_w, double* data_im);
@@ -302,11 +304,11 @@ __global__ void col2im_nd_gpu_kernel(const int n, const Dtype* data_col,
CUDA_KERNEL_LOOP(index, n) {
// Initialize channel_in, computed in the loop below, with intermediate
// computations used to compute the spatial indices.
- int channel_im = index;
+ int c_im = index;
// Calculate d_im (image dimensions).
for (int i = num_axes - 1; i >= 0; --i) {
- d_im[i] = channel_im % im_shape[i + 1] + pad[i];
- channel_im /= im_shape[i + 1];
+ d_im[i] = c_im % im_shape[i + 1] + pad[i];
+ c_im /= im_shape[i + 1];
}
// Calculate col start/end indices.
bool done = false;
@@ -338,7 +340,7 @@ __global__ void col2im_nd_gpu_kernel(const int n, const Dtype* data_col,
(d_im[i] - d_col_iter[i] * stride[i]) * kernel_shape_prod;
kernel_shape_prod *= kernel_shape[i];
}
- final_offset += kernel_shape_prod * channel_im;
+ final_offset += kernel_shape_prod * c_im;
for (int i = 0; i < num_axes; ++i) {
final_offset *= col_shape[i + 1];
final_offset += d_col_iter[i];
diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp
index f2b1dd9..835d2d4 100644
--- a/src/caffe/util/io.cpp
+++ b/src/caffe/util/io.cpp
@@ -2,8 +2,8 @@
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/text_format.h>
-#include <opencv2/core/core.hpp>
#ifdef USE_OPENCV
+#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/highgui/highgui_c.h>
#include <opencv2/imgproc/imgproc.hpp>
diff --git a/src/caffe/util/math_functions.cpp b/src/caffe/util/math_functions.cpp
index 0aab6b1..71c0227 100644
--- a/src/caffe/util/math_functions.cpp
+++ b/src/caffe/util/math_functions.cpp
@@ -349,28 +349,6 @@ template
double caffe_cpu_dot<double>(const int n, const double* x, const double* y);
template <>
-int caffe_cpu_hamming_distance<float>(const int n, const float* x,
- const float* y) {
- int dist = 0;
- for (int i = 0; i < n; ++i) {
- dist += __builtin_popcount(static_cast<uint32_t>(x[i]) ^
- static_cast<uint32_t>(y[i]));
- }
- return dist;
-}
-
-template <>
-int caffe_cpu_hamming_distance<double>(const int n, const double* x,
- const double* y) {
- int dist = 0;
- for (int i = 0; i < n; ++i) {
- dist += __builtin_popcountl(static_cast<uint64_t>(x[i]) ^
- static_cast<uint64_t>(y[i]));
- }
- return dist;
-}
-
-template <>
float caffe_cpu_asum<float>(const int n, const float* x) {
return cblas_sasum(n, x, 1);
}
diff --git a/src/caffe/util/math_functions.cu b/src/caffe/util/math_functions.cu
index 2631a07..4c58753 100644
--- a/src/caffe/util/math_functions.cu
+++ b/src/caffe/util/math_functions.cu
@@ -4,8 +4,6 @@
#include <thrust/reduce.h>
#include <cmath>
-#include <cstdlib>
-#include <cstring>
#include "caffe/common.hpp"
#include "caffe/util/math_functions.hpp"
@@ -373,51 +371,6 @@ DEFINE_AND_INSTANTIATE_GPU_UNARY_FUNC(sign, y[index] = (Dtype(0) < x[index])
- (x[index] < Dtype(0)));
DEFINE_AND_INSTANTIATE_GPU_UNARY_FUNC(sgnbit, y[index] = signbit(x[index]));
-__global__ void popc_kernel(const int n, const float* a,
- const float* b, uint8_t* y) {
- CUDA_KERNEL_LOOP(index, n) {
- y[index] = __popc(static_cast<uint32_t>(a[index]) ^
- static_cast<uint32_t>(b[index]));
- }
-}
-
-__global__ void popcll_kernel(const int n, const double* a,
- const double* b, uint8_t* y) {
- CUDA_KERNEL_LOOP(index, n) {
- y[index] = __popcll(static_cast<uint64_t>(a[index]) ^
- static_cast<uint64_t>(b[index]));
- }
-}
-
-template <>
-uint32_t caffe_gpu_hamming_distance<float>(const int n, const float* x,
- const float* y) {
- // TODO: Fix caffe_gpu_hamming_distance (see failing unit test
- // TestHammingDistanceGPU in test_math_functions.cpp).
- NOT_IMPLEMENTED;
- thrust::device_vector<uint8_t> popcounts(n);
- // NOLINT_NEXT_LINE(whitespace/operators)
- popc_kernel<<<CAFFE_GET_BLOCKS(n), CAFFE_CUDA_NUM_THREADS>>>(
- n, x, y, thrust::raw_pointer_cast(popcounts.data()));
- return thrust::reduce(popcounts.begin(), popcounts.end(),
- (uint32_t) 0, thrust::plus<uint32_t>());
-}
-
-template <>
-uint32_t caffe_gpu_hamming_distance<double>(const int n, const double* x,
- const double* y) {
- // TODO: Fix caffe_gpu_hamming_distance (see failing unit test
- // TestHammingDistanceGPU in test_math_functions.cpp).
- NOT_IMPLEMENTED;
- thrust::device_vector<uint8_t> popcounts(n);
- // NOLINT_NEXT_LINE(whitespace/operators)
- popcll_kernel<<<CAFFE_GET_BLOCKS(n), CAFFE_CUDA_NUM_THREADS>>>(
- n, x, y, thrust::raw_pointer_cast(popcounts.data()));
- return thrust::reduce(popcounts.begin(), popcounts.end(),
- /* NOLINT_NEXT_LINE(build/include_what_you_use) */
- (uint32_t) 0, thrust::plus<uint32_t>());
-}
-
void caffe_gpu_rng_uniform(const int n, unsigned int* r) {
CURAND_CHECK(curandGenerate(Caffe::curand_generator(), r, n));
}
diff --git a/src/caffe/util/upgrade_proto.cpp b/src/caffe/util/upgrade_proto.cpp
index ac379e5..ff3f8ff 100644
--- a/src/caffe/util/upgrade_proto.cpp
+++ b/src/caffe/util/upgrade_proto.cpp
@@ -16,6 +16,67 @@ bool NetNeedsUpgrade(const NetParameter& net_param) {
return NetNeedsV0ToV1Upgrade(net_param) || NetNeedsV1ToV2Upgrade(net_param);
}
+bool UpgradeNetAsNeeded(const string& param_file, NetParameter* param) {
+ bool success = true;
+ if (NetNeedsV0ToV1Upgrade(*param)) {
+ // NetParameter was specified using the old style (V0LayerParameter); try to
+ // upgrade it.
+ LOG(INFO) << "Attempting to upgrade input file specified using deprecated "
+ << "V0LayerParameter: " << param_file;
+ NetParameter original_param(*param);
+ if (!UpgradeV0Net(original_param, param)) {
+ success = false;
+ LOG(ERROR) << "Warning: had one or more problems upgrading "
+ << "V0NetParameter to NetParameter (see above); continuing anyway.";
+ } else {
+ LOG(INFO) << "Successfully upgraded file specified using deprecated "
+ << "V0LayerParameter";
+ }
+ LOG(WARNING) << "Note that future Caffe releases will not support "
+ << "V0NetParameter; use ./build/tools/upgrade_net_proto_text for "
+ << "prototxt and ./build/tools/upgrade_net_proto_binary for model "
+ << "weights upgrade this and any other net protos to the new format.";
+ }
+ // NetParameter uses old style data transformation fields; try to upgrade it.
+ if (NetNeedsDataUpgrade(*param)) {
+ LOG(INFO) << "Attempting to upgrade input file specified using deprecated "
+ << "transformation parameters: " << param_file;
+ UpgradeNetDataTransformation(param);
+ LOG(INFO) << "Successfully upgraded file specified using deprecated "
+ << "data transformation parameters.";
+ LOG(WARNING) << "Note that future Caffe releases will only support "
+ << "transform_param messages for transformation fields.";
+ }
+ if (NetNeedsV1ToV2Upgrade(*param)) {
+ LOG(INFO) << "Attempting to upgrade input file specified using deprecated "
+ << "V1LayerParameter: " << param_file;
+ NetParameter original_param(*param);
+ if (!UpgradeV1Net(original_param, param)) {
+ success = false;
+ LOG(ERROR) << "Warning: had one or more problems upgrading "
+ << "V1LayerParameter (see above); continuing anyway.";
+ } else {
+ LOG(INFO) << "Successfully upgraded file specified using deprecated "
+ << "V1LayerParameter";
+ }
+ }
+ return success;
+}
+
+void ReadNetParamsFromTextFileOrDie(const string& param_file,
+ NetParameter* param) {
+ CHECK(ReadProtoFromTextFile(param_file, param))
+ << "Failed to parse NetParameter file: " << param_file;
+ UpgradeNetAsNeeded(param_file, param);
+}
+
+void ReadNetParamsFromBinaryFileOrDie(const string& param_file,
+ NetParameter* param) {
+ CHECK(ReadProtoFromBinaryFile(param_file, param))
+ << "Failed to parse NetParameter file: " << param_file;
+ UpgradeNetAsNeeded(param_file, param);
+}
+
bool NetNeedsV0ToV1Upgrade(const NetParameter& net_param) {
for (int i = 0; i < net_param.layers_size(); ++i) {
if (net_param.layers(i).has_layer()) {
@@ -583,53 +644,6 @@ void UpgradeNetDataTransformation(NetParameter* net_param) {
}
}
-bool UpgradeNetAsNeeded(const string& param_file, NetParameter* param) {
- bool success = true;
- if (NetNeedsV0ToV1Upgrade(*param)) {
- // NetParameter was specified using the old style (V0LayerParameter); try to
- // upgrade it.
- LOG(INFO) << "Attempting to upgrade input file specified using deprecated "
- << "V0LayerParameter: " << param_file;
- NetParameter original_param(*param);
- if (!UpgradeV0Net(original_param, param)) {
- success = false;
- LOG(ERROR) << "Warning: had one or more problems upgrading "
- << "V0NetParameter to NetParameter (see above); continuing anyway.";
- } else {
- LOG(INFO) << "Successfully upgraded file specified using deprecated "
- << "V0LayerParameter";
- }
- LOG(WARNING) << "Note that future Caffe releases will not support "
- << "V0NetParameter; use ./build/tools/upgrade_net_proto_text for "
- << "prototxt and ./build/tools/upgrade_net_proto_binary for model "
- << "weights upgrade this and any other net protos to the new format.";
- }
- // NetParameter uses old style data transformation fields; try to upgrade it.
- if (NetNeedsDataUpgrade(*param)) {
- LOG(INFO) << "Attempting to upgrade input file specified using deprecated "
- << "transformation parameters: " << param_file;
- UpgradeNetDataTransformation(param);
- LOG(INFO) << "Successfully upgraded file specified using deprecated "
- << "data transformation parameters.";
- LOG(WARNING) << "Note that future Caffe releases will only support "
- << "transform_param messages for transformation fields.";
- }
- if (NetNeedsV1ToV2Upgrade(*param)) {
- LOG(INFO) << "Attempting to upgrade input file specified using deprecated "
- << "V1LayerParameter: " << param_file;
- NetParameter original_param(*param);
- if (!UpgradeV1Net(original_param, param)) {
- success = false;
- LOG(ERROR) << "Warning: had one or more problems upgrading "
- << "V1LayerParameter (see above); continuing anyway.";
- } else {
- LOG(INFO) << "Successfully upgraded file specified using deprecated "
- << "V1LayerParameter";
- }
- }
- return success;
-}
-
bool UpgradeV1Net(const NetParameter& v1_net_param, NetParameter* net_param) {
bool is_fully_compatible = true;
if (v1_net_param.layer_size() > 0) {
@@ -923,18 +937,78 @@ const char* UpgradeV1LayerType(const V1LayerParameter_LayerType type) {
}
}
-void ReadNetParamsFromTextFileOrDie(const string& param_file,
- NetParameter* param) {
- CHECK(ReadProtoFromTextFile(param_file, param))
- << "Failed to parse NetParameter file: " << param_file;
- UpgradeNetAsNeeded(param_file, param);
+// Return true iff the solver contains any old solver_type specified as enums
+bool SolverNeedsTypeUpgrade(const SolverParameter& solver_param) {
+ if (solver_param.has_solver_type()) {
+ return true;
+ }
+ return false;
}
-void ReadNetParamsFromBinaryFileOrDie(const string& param_file,
- NetParameter* param) {
- CHECK(ReadProtoFromBinaryFile(param_file, param))
- << "Failed to parse NetParameter file: " << param_file;
- UpgradeNetAsNeeded(param_file, param);
+bool UpgradeSolverType(SolverParameter* solver_param) {
+ CHECK(!solver_param->has_solver_type() || !solver_param->has_type())
+ << "Failed to upgrade solver: old solver_type field (enum) and new type "
+ << "field (string) cannot be both specified in solver proto text.";
+ if (solver_param->has_solver_type()) {
+ string type;
+ switch (solver_param->solver_type()) {
+ case SolverParameter_SolverType_SGD:
+ type = "SGD";
+ break;
+ case SolverParameter_SolverType_NESTEROV:
+ type = "Nesterov";
+ break;
+ case SolverParameter_SolverType_ADAGRAD:
+ type = "AdaGrad";
+ break;
+ case SolverParameter_SolverType_RMSPROP:
+ type = "RMSProp";
+ break;
+ case SolverParameter_SolverType_ADADELTA:
+ type = "AdaDelta";
+ break;
+ case SolverParameter_SolverType_ADAM:
+ type = "Adam";
+ break;
+ default:
+ LOG(FATAL) << "Unknown SolverParameter solver_type: " << type;
+ }
+ solver_param->set_type(type);
+ solver_param->clear_solver_type();
+ } else {
+ LOG(ERROR) << "Warning: solver type already up to date. ";
+ return false;
+ }
+ return true;
+}
+
+// Check for deprecations and upgrade the SolverParameter as needed.
+bool UpgradeSolverAsNeeded(const string& param_file, SolverParameter* param) {
+ bool success = true;
+ // Try to upgrade old style solver_type enum fields into new string type
+ if (SolverNeedsTypeUpgrade(*param)) {
+ LOG(INFO) << "Attempting to upgrade input file specified using deprecated "
+ << "'solver_type' field (enum)': " << param_file;
+ if (!UpgradeSolverType(param)) {
+ success = false;
+ LOG(ERROR) << "Warning: had one or more problems upgrading "
+ << "SolverType (see above).";
+ } else {
+ LOG(INFO) << "Successfully upgraded file specified using deprecated "
+ << "'solver_type' field (enum) to 'type' field (string).";
+ LOG(WARNING) << "Note that future Caffe releases will only support "
+ << "'type' field (string) for a solver's type.";
+ }
+ }
+ return success;
+}
+
+// Read parameters from a file into a SolverParameter proto message.
+void ReadSolverParamsFromTextFileOrDie(const string& param_file,
+ SolverParameter* param) {
+ CHECK(ReadProtoFromTextFile(param_file, param))
+ << "Failed to parse SolverParameter file: " << param_file;
+ UpgradeSolverAsNeeded(param_file, param);
}
} // namespace caffe
diff --git a/tools/caffe.cpp b/tools/caffe.cpp
index e3f684b..305cfc3 100644
--- a/tools/caffe.cpp
+++ b/tools/caffe.cpp
@@ -157,7 +157,7 @@ int train() {
"but not both.";
caffe::SolverParameter solver_param;
- caffe::ReadProtoFromTextFileOrDie(FLAGS_solver, &solver_param);
+ caffe::ReadSolverParamsFromTextFileOrDie(FLAGS_solver, &solver_param);
// If the gpus flag is not provided, allow the mode and device to be set
// in the solver prototxt.
@@ -194,7 +194,7 @@ int train() {
GetRequestedAction(FLAGS_sighup_effect));
shared_ptr<caffe::Solver<float> >
- solver(caffe::GetSolver<float>(solver_param));
+ solver(caffe::SolverRegistry<float>::CreateSolver(solver_param));
solver->SetActionFunction(signal_handler.GetActionFunction());
diff --git a/tools/convert_imageset.cpp b/tools/convert_imageset.cpp
index e51a263..9c52bfa 100644
--- a/tools/convert_imageset.cpp
+++ b/tools/convert_imageset.cpp
@@ -20,6 +20,7 @@
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/db.hpp"
+#include "caffe/util/format.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/rng.hpp"
@@ -99,8 +100,6 @@ int main(int argc, char** argv) {
std::string root_folder(argv[1]);
Datum datum;
int count = 0;
- const int kMaxKeyLength = 256;
- char key_cstr[kMaxKeyLength];
int data_size = 0;
bool data_size_initialized = false;
@@ -131,13 +130,12 @@ int main(int argc, char** argv) {
}
}
// sequential
- int length = snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id,
- lines[line_id].first.c_str());
+ string key_str = caffe::format_int(line_id, 8) + "_" + lines[line_id].first;
// Put in db
string out;
CHECK(datum.SerializeToString(&out));
- txn->Put(string(key_cstr, length), out);
+ txn->Put(key_str, out);
if (++count % 1000 == 0) {
// Commit db
diff --git a/tools/extra/parse_log.sh b/tools/extra/parse_log.sh
index 98ef0a0..9892c89 100755
--- a/tools/extra/parse_log.sh
+++ b/tools/extra/parse_log.sh
@@ -14,7 +14,12 @@ echo "Usage parse_log.sh /path/to/your.log"
exit
fi
LOG=`basename $1`
-grep -B 1 'Test ' $1 > aux.txt
+sed -n '/Iteration .* Testing net/,/Iteration *. loss/p' $1 > aux.txt
+sed -i '/Waiting for data/d' aux.txt
+sed -i '/prefetch queue empty/d' aux.txt
+sed -i '/Iteration .* loss/d' aux.txt
+sed -i '/Iteration .* lr/d' aux.txt
+sed -i '/Train net/d' aux.txt
grep 'Iteration ' aux.txt | sed 's/.*Iteration \([[:digit:]]*\).*/\1/g' > aux0.txt
grep 'Test net output #0' aux.txt | awk '{print $11}' > aux1.txt
grep 'Test net output #1' aux.txt | awk '{print $11}' > aux2.txt
diff --git a/tools/extra/plot_training_log.py.example b/tools/extra/plot_training_log.py.example
index b6fda54..4d3ed0d 100755
--- a/tools/extra/plot_training_log.py.example
+++ b/tools/extra/plot_training_log.py.example
@@ -150,7 +150,7 @@ Be warned that the fields in the training log may change in the future.
You had better check the data files and change the mapping from field name to
field index in create_field_index before designing your own plots.
Usage:
- ./plot_log.sh chart_type[0-%s] /where/to/save.png /path/to/first.log ...
+ ./plot_training_log.py chart_type[0-%s] /where/to/save.png /path/to/first.log ...
Notes:
1. Supporting multiple logs.
2. Log file name must end with the lower-cased "%s".
diff --git a/tools/extra/summarize.py b/tools/extra/summarize.py
new file mode 100755
index 0000000..7e2d22f
--- /dev/null
+++ b/tools/extra/summarize.py
@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+
+"""Net summarization tool.
+
+This tool summarizes the structure of a net in a concise but comprehensive
+tabular listing, taking a prototxt file as input.
+
+Use this tool to check at a glance that the computation you've specified is the
+computation you expect.
+"""
+
+from caffe.proto import caffe_pb2
+from google import protobuf
+import re
+import argparse
+
+# ANSI codes for coloring blobs (used cyclically)
+COLORS = ['92', '93', '94', '95', '97', '96', '42', '43;30', '100',
+ '444', '103;30', '107;30']
+DISCONNECTED_COLOR = '41'
+
+def read_net(filename):
+ net = caffe_pb2.NetParameter()
+ with open(filename) as f:
+ protobuf.text_format.Parse(f.read(), net)
+ return net
+
+def format_param(param):
+ out = []
+ if len(param.name) > 0:
+ out.append(param.name)
+ if param.lr_mult != 1:
+ out.append('x{}'.format(param.lr_mult))
+ if param.decay_mult != 1:
+ out.append('Dx{}'.format(param.decay_mult))
+ return ' '.join(out)
+
+def printed_len(s):
+ return len(re.sub(r'\033\[[\d;]+m', '', s))
+
+def print_table(table, max_width):
+ """Print a simple nicely-aligned table.
+
+ table must be a list of (equal-length) lists. Columns are space-separated,
+ and as narrow as possible, but no wider than max_width. Text may overflow
+ columns; note that unlike string.format, this will not affect subsequent
+ columns, if possible."""
+
+ max_widths = [max_width] * len(table[0])
+ column_widths = [max(printed_len(row[j]) + 1 for row in table)
+ for j in range(len(table[0]))]
+ column_widths = [min(w, max_w) for w, max_w in zip(column_widths, max_widths)]
+
+ for row in table:
+ row_str = ''
+ right_col = 0
+ for cell, width in zip(row, column_widths):
+ right_col += width
+ row_str += cell + ' '
+ row_str += ' ' * max(right_col - printed_len(row_str), 0)
+ print row_str
+
+def summarize_net(net):
+ disconnected_tops = set()
+ for lr in net.layer:
+ disconnected_tops |= set(lr.top)
+ disconnected_tops -= set(lr.bottom)
+
+ table = []
+ colors = {}
+ for lr in net.layer:
+ tops = []
+ for ind, top in enumerate(lr.top):
+ color = colors.setdefault(top, COLORS[len(colors) % len(COLORS)])
+ if top in disconnected_tops:
+ top = '\033[1;4m' + top
+ if len(lr.loss_weight) > 0:
+ top = '{} * {}'.format(lr.loss_weight[ind], top)
+ tops.append('\033[{}m{}\033[0m'.format(color, top))
+ top_str = ', '.join(tops)
+
+ bottoms = []
+ for bottom in lr.bottom:
+ color = colors.get(bottom, DISCONNECTED_COLOR)
+ bottoms.append('\033[{}m{}\033[0m'.format(color, bottom))
+ bottom_str = ', '.join(bottoms)
+
+ if lr.type == 'Python':
+ type_str = lr.python_param.module + '.' + lr.python_param.layer
+ else:
+ type_str = lr.type
+
+ # Summarize conv/pool parameters.
+ # TODO support rectangular/ND parameters
+ conv_param = lr.convolution_param
+ if (lr.type in ['Convolution', 'Deconvolution']
+ and len(conv_param.kernel_size) == 1):
+ arg_str = str(conv_param.kernel_size[0])
+ if len(conv_param.stride) > 0 and conv_param.stride[0] != 1:
+ arg_str += '/' + str(conv_param.stride[0])
+ if len(conv_param.pad) > 0 and conv_param.pad[0] != 0:
+ arg_str += '+' + str(conv_param.pad[0])
+ arg_str += ' ' + str(conv_param.num_output)
+ if conv_param.group != 1:
+ arg_str += '/' + str(conv_param.group)
+ elif lr.type == 'Pooling':
+ arg_str = str(lr.pooling_param.kernel_size)
+ if lr.pooling_param.stride != 1:
+ arg_str += '/' + str(lr.pooling_param.stride)
+ if lr.pooling_param.pad != 0:
+ arg_str += '+' + str(lr.pooling_param.pad)
+ else:
+ arg_str = ''
+
+ if len(lr.param) > 0:
+ param_strs = map(format_param, lr.param)
+ if max(map(len, param_strs)) > 0:
+ param_str = '({})'.format(', '.join(param_strs))
+ else:
+ param_str = ''
+ else:
+ param_str = ''
+
+ table.append([lr.name, type_str, param_str, bottom_str, '->', top_str,
+ arg_str])
+ return table
+
+def main():
+ parser = argparse.ArgumentParser(description="Print a concise summary of net computation.")
+ parser.add_argument('filename', help='net prototxt file to summarize')
+ parser.add_argument('-w', '--max-width', help='maximum field width',
+ type=int, default=30)
+ args = parser.parse_args()
+
+ net = read_net(args.filename)
+ table = summarize_net(net)
+ print_table(table, max_width=args.max_width)
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/extract_features.cpp b/tools/extract_features.cpp
index 084c9bf..d6562f9 100644
--- a/tools/extract_features.cpp
+++ b/tools/extract_features.cpp
@@ -1,4 +1,3 @@
-#include <stdio.h> // for snprintf
#include <string>
#include <vector>
@@ -10,14 +9,13 @@
#include "caffe/net.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/db.hpp"
+#include "caffe/util/format.hpp"
#include "caffe/util/io.hpp"
-#include "caffe/vision_layers.hpp"
using caffe::Blob;
using caffe::Caffe;
using caffe::Datum;
using caffe::Net;
-using boost::shared_ptr;
using std::string;
namespace db = caffe::db;
@@ -52,7 +50,7 @@ int feature_extraction_pipeline(int argc, char** argv) {
arg_pos = num_required_args;
if (argc > arg_pos && strcmp(argv[arg_pos], "GPU") == 0) {
LOG(ERROR)<< "Using GPU";
- uint device_id = 0;
+ int device_id = 0;
if (argc > arg_pos + 1) {
device_id = atoi(argv[arg_pos + 1]);
CHECK_GE(device_id, 0);
@@ -96,7 +94,7 @@ int feature_extraction_pipeline(int argc, char** argv) {
}
*/
std::string feature_extraction_proto(argv[++arg_pos]);
- shared_ptr<Net<Dtype> > feature_extraction_net(
+ boost::shared_ptr<Net<Dtype> > feature_extraction_net(
new Net<Dtype>(feature_extraction_proto, caffe::TEST));
feature_extraction_net->CopyTrainedLayersFrom(pretrained_binary_proto);
@@ -120,30 +118,28 @@ int feature_extraction_pipeline(int argc, char** argv) {
int num_mini_batches = atoi(argv[++arg_pos]);
- std::vector<shared_ptr<db::DB> > feature_dbs;
- std::vector<shared_ptr<db::Transaction> > txns;
+ std::vector<boost::shared_ptr<db::DB> > feature_dbs;
+ std::vector<boost::shared_ptr<db::Transaction> > txns;
const char* db_type = argv[++arg_pos];
for (size_t i = 0; i < num_features; ++i) {
LOG(INFO)<< "Opening dataset " << dataset_names[i];
- shared_ptr<db::DB> db(db::GetDB(db_type));
+ boost::shared_ptr<db::DB> db(db::GetDB(db_type));
db->Open(dataset_names.at(i), db::NEW);
feature_dbs.push_back(db);
- shared_ptr<db::Transaction> txn(db->NewTransaction());
+ boost::shared_ptr<db::Transaction> txn(db->NewTransaction());
txns.push_back(txn);
}
LOG(ERROR)<< "Extacting Features";
Datum datum;
- const int kMaxKeyStrLength = 100;
- char key_str[kMaxKeyStrLength];
std::vector<Blob<float>*> input_vec;
std::vector<int> image_indices(num_features, 0);
for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index) {
feature_extraction_net->Forward(input_vec);
for (int i = 0; i < num_features; ++i) {
- const shared_ptr<Blob<Dtype> > feature_blob = feature_extraction_net
- ->blob_by_name(blob_names[i]);
+ const boost::shared_ptr<Blob<Dtype> > feature_blob =
+ feature_extraction_net->blob_by_name(blob_names[i]);
int batch_size = feature_blob->num();
int dim_features = feature_blob->count() / batch_size;
const Dtype* feature_blob_data;
@@ -158,11 +154,11 @@ int feature_extraction_pipeline(int argc, char** argv) {
for (int d = 0; d < dim_features; ++d) {
datum.add_float_data(feature_blob_data[d]);
}
- int length = snprintf(key_str, kMaxKeyStrLength, "%010d",
- image_indices[i]);
+ string key_str = caffe::format_int(image_indices[i], 10);
+
string out;
CHECK(datum.SerializeToString(&out));
- txns.at(i)->Put(std::string(key_str, length), out);
+ txns.at(i)->Put(key_str, out);
++image_indices[i];
if (image_indices[i] % 1000 == 0) {
txns.at(i)->Commit();
@@ -186,4 +182,3 @@ int feature_extraction_pipeline(int argc, char** argv) {
LOG(ERROR)<< "Successfully extracted the features!";
return 0;
}
-
diff --git a/tools/upgrade_solver_proto_text.cpp b/tools/upgrade_solver_proto_text.cpp
new file mode 100644
index 0000000..7130232
--- /dev/null
+++ b/tools/upgrade_solver_proto_text.cpp
@@ -0,0 +1,50 @@
+// This is a script to upgrade old solver prototxts to the new format.
+// Usage:
+// upgrade_solver_proto_text old_solver_proto_file_in solver_proto_file_out
+
+#include <cstring>
+#include <fstream> // NOLINT(readability/streams)
+#include <iostream> // NOLINT(readability/streams)
+#include <string>
+
+#include "caffe/caffe.hpp"
+#include "caffe/util/io.hpp"
+#include "caffe/util/upgrade_proto.hpp"
+
+using std::ofstream;
+
+using namespace caffe; // NOLINT(build/namespaces)
+
+int main(int argc, char** argv) {
+ ::google::InitGoogleLogging(argv[0]);
+ if (argc != 3) {
+ LOG(ERROR) << "Usage: upgrade_solver_proto_text "
+ << "old_solver_proto_file_in solver_proto_file_out";
+ return 1;
+ }
+
+ SolverParameter solver_param;
+ string input_filename(argv[1]);
+ if (!ReadProtoFromTextFile(input_filename, &solver_param)) {
+ LOG(ERROR) << "Failed to parse input text file as SolverParameter: "
+ << input_filename;
+ return 2;
+ }
+ bool need_upgrade = SolverNeedsTypeUpgrade(solver_param);
+ bool success = true;
+ if (need_upgrade) {
+ success = UpgradeSolverAsNeeded(input_filename, &solver_param);
+ if (!success) {
+ LOG(ERROR) << "Encountered error(s) while upgrading prototxt; "
+ << "see details above.";
+ }
+ } else {
+ LOG(ERROR) << "File already in latest proto format: " << input_filename;
+ }
+
+ // Save new format prototxt.
+ WriteProtoToTextFile(solver_param, argv[2]);
+
+ LOG(ERROR) << "Wrote upgraded SolverParameter text proto to " << argv[2];
+ return !success;
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/caffe-contrib.git
More information about the debian-science-commits
mailing list