Fixing up maxpool layer

Makefile

@@ -7,7 +7,6 @@ else
 endif
 UNAME = $(shell uname)
 OPTS=-Ofast -flto
-OPTS=-Ofast -flto
 ifeq ($(UNAME), Darwin)
 COMMON+= -isystem /usr/local/Cellar/opencv/2.4.6.1/include/opencv -isystem /usr/local/Cellar/opencv/2.4.6.1/include
 ifeq ($(GPU), 1)

src/cnn.c

@@ -278,29 +278,20 @@ void test_data()
 	free_data(train);
 }
 
-void train_full()
+void train_assira()
 {
-	network net = parse_network_cfg("cfg/imagenet.cfg");
+	network net = parse_network_cfg("cfg/assira.cfg");
 	srand(2222222);
 	int i = 0;
 	char *labels[] = {"cat","dog"};
-	float lr = .00001;
-	float momentum = .9;
-	float decay = 0.01;
 	while(1){
 		i += 1000;
-		data train = load_data_image_pathfile_random("images/assira/train.list", 1000, labels, 2, 256, 256);
-		//image im = float_to_image(256, 256, 3,train.X.vals[0]);
-		//visualize_network(net);
-		//cvWaitKey(100);
-		//show_image(im, "input");
-		//cvWaitKey(100);
-		//scale_data_rows(train, 1./255.);
+		data train = load_data_image_pathfile_random("data/assira/train.list", 1000, labels, 2, 256, 256);
 		normalize_data_rows(train);
 		clock_t start = clock(), end;
-		float loss = train_network_sgd(net, train, 1000);
+		float loss = train_network_sgd_gpu(net, train, 10);
 		end = clock();
-		printf("%d: %f, Time: %lf seconds, LR: %f, Momentum: %f, Decay: %f\n", i, loss, (float)(end-start)/CLOCKS_PER_SEC, lr, momentum, decay);
+		printf("%d: %f, Time: %lf seconds\n", i, loss, (float)(end-start)/CLOCKS_PER_SEC );
 		free_data(train);
 		if(i%10000==0){
 			char buff[256];
@@ -367,10 +358,10 @@ void train_cifar10()
     data train = load_all_cifar10();
     while(++count <= 10000){
         clock_t start = clock(), end;
-        float loss = train_network_sgd(net, train, iters);
+        float loss = train_network_sgd_gpu(net, train, iters);
         end = clock();
-        visualize_network(net);
-        cvWaitKey(5000);
+        //visualize_network(net);
+        //cvWaitKey(5000);
 
         //float test_acc = network_accuracy(net, test);
         //printf("%d: Loss: %f, Test Acc: %f, Time: %lf seconds, LR: %f, Momentum: %f, Decay: %f\n", count, loss, test_acc,(float)(end-start)/CLOCKS_PER_SEC, net.learning_rate, net.momentum, net.decay);
@@ -902,7 +893,7 @@ void test_distribution()
 
 int main(int argc, char *argv[])
 {
-    //train_full();
+    //train_assira();
     //test_distribution();
     //feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW);
 

src/connected_layer.c

@@ -38,9 +38,17 @@ connected_layer *make_connected_layer(int batch, int inputs, int outputs, ACTIVA
     for(i = 0; i < outputs; ++i){
         //layer->biases[i] = rand_normal()*scale + scale;
         layer->biases[i] = 1;
-        }
+    }
 
     #ifdef GPU
+    layer->weights_cl = cl_make_array(layer->weights, inputs*outputs);
+    layer->biases_cl = cl_make_array(layer->biases, outputs);
+
+    layer->weight_updates_cl = cl_make_array(layer->weight_updates, inputs*outputs);
+    layer->bias_updates_cl = cl_make_array(layer->bias_updates, outputs);
+
+    layer->output_cl = cl_make_array(layer->output, outputs*batch);
+    layer->delta_cl = cl_make_array(layer->delta, outputs*batch);
     #endif
     layer->activation = activation;
     return layer;
@@ -76,8 +84,8 @@ void backward_connected_layer(connected_layer layer, float *input, float *delta)
 {
     int i;
     gradient_array(layer.output, layer.outputs*layer.batch, layer.activation, layer.delta);
-    for(i = 0; i < layer.outputs*layer.batch; ++i){
-        layer.bias_updates[i%layer.outputs] += layer.delta[i];
+    for(i = 0; i < layer.batch; ++i){
+        axpy_cpu(layer.outputs, 1, layer.delta + i*layer.outputs, 1, layer.bias_updates, 1);
     }
     int m = layer.inputs;
     int k = layer.batch;
@@ -98,3 +106,61 @@ void backward_connected_layer(connected_layer layer, float *input, float *delta)
     if(c) gemm(0,1,m,n,k,1,a,k,b,k,0,c,n);
 }
 
+#ifdef GPU
+
+void update_connected_layer_gpu(connected_layer layer)
+{
+    axpy_ongpu(layer.outputs, layer.learning_rate, layer.bias_updates_cl, 1, layer.biases_cl, 1);
+    scal_ongpu(layer.outputs, layer.momentum, layer.bias_updates_cl, 1);
+
+    scal_ongpu(layer.inputs*layer.outputs, 1.-layer.learning_rate*layer.decay, layer.weights_cl, 1);
+    axpy_ongpu(layer.inputs*layer.outputs, layer.learning_rate, layer.weight_updates_cl, 1, layer.weights_cl, 1);
+    scal_ongpu(layer.inputs*layer.outputs, layer.momentum, layer.weight_updates_cl, 1);
+}
+
+void forward_connected_layer_gpu(connected_layer layer, cl_mem input)
+{
+    int i;
+    for(i = 0; i < layer.batch; ++i){
+        cl_mem sub = cl_sub_array(layer.output_cl, i*layer.outputs, layer.outputs);
+        copy_ongpu(layer.outputs, layer.biases_cl, 1, sub, 1);
+        clReleaseMemObject(sub);
+    }
+    int m = layer.batch;
+    int k = layer.inputs;
+    int n = layer.outputs;
+    cl_mem a = input;
+    cl_mem b = layer.weights_cl;
+    cl_mem c = layer.output_cl;
+    gemm_ongpu(0,0,m,n,k,1,a,k,b,n,1,c,n);
+    activate_array_ongpu(layer.output_cl, layer.outputs*layer.batch, layer.activation);
+}
+
+void backward_connected_layer_gpu(connected_layer layer, cl_mem input, cl_mem delta)
+{
+    int i;
+    gradient_array_ongpu(layer.output_cl, layer.outputs*layer.batch, layer.activation, layer.delta_cl);
+    for(i = 0; i < layer.batch; ++i){
+        cl_mem sub = cl_sub_array(layer.delta_cl, i*layer.outputs, layer.outputs);
+        axpy_ongpu(layer.outputs, 1, sub, 1, layer.bias_updates_cl, 1);
+        clReleaseMemObject(sub);
+    }
+    int m = layer.inputs;
+    int k = layer.batch;
+    int n = layer.outputs;
+    cl_mem a = input;
+    cl_mem b = layer.delta_cl;
+    cl_mem c = layer.weight_updates_cl;
+    gemm_ongpu(1,0,m,n,k,1,a,m,b,n,1,c,n);
+
+    m = layer.batch;
+    k = layer.outputs;
+    n = layer.inputs;
+
+    a = layer.delta_cl;
+    b = layer.weights_cl;
+    c = delta;
+
+    if(c) gemm_ongpu(0,1,m,n,k,1,a,k,b,k,0,c,n);
+}
+    #endif

src/connected_layer.h

@@ -31,9 +31,6 @@ typedef struct{
     cl_mem weight_updates_cl;
     cl_mem bias_updates_cl;
 
-    cl_mem weight_momentum_cl;
-    cl_mem bias_momentum_cl;
-
     cl_mem output_cl;
     cl_mem delta_cl;
     #endif
@@ -47,6 +44,11 @@ void forward_connected_layer(connected_layer layer, float *input);
 void backward_connected_layer(connected_layer layer, float *input, float *delta);
 void update_connected_layer(connected_layer layer);
 
+#ifdef GPU
+void forward_connected_layer_gpu(connected_layer layer, cl_mem input);
+void backward_connected_layer_gpu(connected_layer layer, cl_mem input, cl_mem delta);
+void update_connected_layer_gpu(connected_layer layer);
+#endif
 
 #endif
 

src/maxpool_layer.c

@@ -27,7 +27,7 @@ maxpool_layer *make_maxpool_layer(int batch, int h, int w, int c, int size, int
     layer->c = c;
     layer->size = size;
     layer->stride = stride;
-    layer->max_indexes = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * c*batch, sizeof(int));
+    layer->indexes = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * c*batch, sizeof(int));
     layer->output = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * c*batch, sizeof(float));
     layer->delta = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * c*batch, sizeof(float));
     return layer;
@@ -44,36 +44,35 @@ void resize_maxpool_layer(maxpool_layer *layer, int h, int w, int c)
 
 void forward_maxpool_layer(const maxpool_layer layer, float *input)
 {
-    int b;
-    for(b = 0; b < layer.batch; ++b){
-        int h = (layer.h-1)/layer.stride + 1;
-        int w = (layer.w-1)/layer.stride + 1;
-        int c = layer.c;
-
-        int i,j,k,l,m;
-        for(k = 0; k < layer.c; ++k){
-            for(i = 0; i < layer.h; i += layer.stride){
-                for(j = 0; j < layer.w; j += layer.stride){
-                    int out_index = j/layer.stride + w*(i/layer.stride + h*(k + c*b));
-                    layer.output[out_index] = -FLT_MAX;
-                    int lower = (-layer.size-1)/2 + 1;
-                    int upper = layer.size/2 + 1;
+    int b,i,j,k,l,m;
+    int w_offset = (-layer.size-1)/2 + 1;
+    int h_offset = (-layer.size-1)/2 + 1;
 
-                    int lh = (i+lower < 0)       ? 0 : i+lower;
-                    int uh = (i+upper > layer.h) ? layer.h : i+upper;
+    int h = (layer.h-1)/layer.stride + 1;
+    int w = (layer.w-1)/layer.stride + 1;
+    int c = layer.c;
 
-                    int lw = (j+lower < 0)       ? 0 : j+lower;
-                    int uw = (j+upper > layer.w) ? layer.w : j+upper;
-                    for(l = lh; l < uh; ++l){
-                        for(m = lw; m < uw; ++m){
-                            //printf("%d %d\n", l, m);
-                            int index = m + layer.w*(l + layer.h*(k + b*layer.c));
-                            if(input[index] > layer.output[out_index]){
-                                layer.output[out_index] = input[index];
-                                layer.max_indexes[out_index] = index;
-                            }
+    for(b = 0; b < layer.batch; ++b){
+        for(k = 0; k < c; ++k){
+            for(i = 0; i < h; ++i){
+                for(j = 0; j < w; ++j){
+                    int out_index = j + w*(i + h*(k + c*b));
+                    float max = -FLT_MAX;
+                    int max_i = -1;
+                    for(l = 0; l < layer.size; ++l){
+                        for(m = 0; m < layer.size; ++m){
+                            int cur_h = h_offset + i*layer.stride + l;
+                            int cur_w = w_offset + j*layer.stride + m;
+                            int index = cur_w + layer.w*(cur_h + layer.h*(k + b*layer.c));
+                            int valid = (cur_h >= 0 && cur_h < layer.h &&
+                                         cur_w >= 0 && cur_w < layer.w);
+                            float val = (valid != 0) ? input[index] : -INFINITY;
+                            max_i = (val > max) ? index : max_i;
+                            max   = (val > max) ? val   : max;
                         }
                     }
+                    layer.output[out_index] = max;
+                    layer.indexes[out_index] = max_i;
                 }
             }
         }
@@ -88,7 +87,7 @@ void backward_maxpool_layer(const maxpool_layer layer, float *input, float *delt
     int c = layer.c;
     memset(delta, 0, layer.batch*layer.h*layer.w*layer.c*sizeof(float));
     for(i = 0; i < h*w*c*layer.batch; ++i){
-        int index = layer.max_indexes[i];
+        int index = layer.indexes[i];
         delta[index] += layer.delta[i];
     }
 }

src/maxpool_layer.h

@@ -8,7 +8,7 @@ typedef struct {
     int h,w,c;
     int stride;
     int size;
-    int *max_indexes;
+    int *indexes;
     float *delta;
     float *output;
 } maxpool_layer;

src/network.c

@@ -24,7 +24,8 @@ network make_network(int n, int batch)
     net.outputs = 0;
     net.output = 0;
     #ifdef GPU
-    net.input_cl = 0;
+    net.input_cl = calloc(1, sizeof(cl_mem));
+    net.truth_cl = calloc(1, sizeof(cl_mem));
     #endif
     return net;
 }
@@ -43,12 +44,12 @@ void forward_network_gpu(network net, cl_mem input, cl_mem truth, int train)
             cost_layer layer = *(cost_layer *)net.layers[i];
             forward_cost_layer_gpu(layer, input, truth);
         }
-        /*
         else if(net.types[i] == CONNECTED){
             connected_layer layer = *(connected_layer *)net.layers[i];
-            forward_connected_layer(layer, input, train);
-            input = layer.output;
+            forward_connected_layer_gpu(layer, input);
+            input = layer.output_cl;
         }
+        /*
         else if(net.types[i] == SOFTMAX){
             softmax_layer layer = *(softmax_layer *)net.layers[i];
             forward_softmax_layer(layer, input);
@@ -94,6 +95,10 @@ void backward_network_gpu(network net, cl_mem input)
             cost_layer layer = *(cost_layer *)net.layers[i];
             backward_cost_layer_gpu(layer, prev_input, prev_delta);
         }
+        else if(net.types[i] == CONNECTED){
+            connected_layer layer = *(connected_layer *)net.layers[i];
+            backward_connected_layer_gpu(layer, prev_input, prev_delta);
+        }
     }
 }
 
@@ -105,18 +110,9 @@ void update_network_gpu(network net)
             convolutional_layer layer = *(convolutional_layer *)net.layers[i];
             update_convolutional_layer_gpu(layer);
         }
-        else if(net.types[i] == MAXPOOL){
-            //maxpool_layer layer = *(maxpool_layer *)net.layers[i];
-        }
-        else if(net.types[i] == SOFTMAX){
-            //maxpool_layer layer = *(maxpool_layer *)net.layers[i];
-        }
-        else if(net.types[i] == NORMALIZATION){
-            //maxpool_layer layer = *(maxpool_layer *)net.layers[i];
-        }
         else if(net.types[i] == CONNECTED){
             connected_layer layer = *(connected_layer *)net.layers[i];
-            update_connected_layer(layer);
+            update_connected_layer_gpu(layer);
         }
     }
 }
@@ -127,6 +123,10 @@ cl_mem get_network_output_cl_layer(network net, int i)
         convolutional_layer layer = *(convolutional_layer *)net.layers[i];
         return layer.output_cl;
     }
+    else if(net.types[i] == CONNECTED){
+        connected_layer layer = *(connected_layer *)net.layers[i];
+        return layer.output_cl;
+    }
     return 0;
 }
 
@@ -136,6 +136,10 @@ cl_mem get_network_delta_cl_layer(network net, int i)
         convolutional_layer layer = *(convolutional_layer *)net.layers[i];
         return layer.delta_cl;
     }
+    else if(net.types[i] == CONNECTED){
+        connected_layer layer = *(connected_layer *)net.layers[i];
+        return layer.delta_cl;
+    }
     return 0;
 }
 
@@ -347,6 +351,46 @@ void backward_network(network net, float *input)
     }
 }
 
+#ifdef GPU
+float train_network_datum_gpu(network net, float *x, float *y)
+{
+    int x_size = get_network_input_size(net)*net.batch;
+    int y_size = get_network_output_size(net)*net.batch;
+    if(!*net.input_cl){
+        *net.input_cl = cl_make_array(x, x_size);
+        *net.truth_cl = cl_make_array(y, y_size);
+    }else{
+        cl_write_array(*net.input_cl, x, x_size);
+        cl_write_array(*net.truth_cl, y, y_size);
+    }
+    forward_network_gpu(net, *net.input_cl, *net.truth_cl, 1);
+    //int class = get_predicted_class_network(net);
+    backward_network_gpu(net, *net.input_cl);
+    float error = get_network_cost(net);
+    update_network_gpu(net);
+    //return (y[class]?1:0);
+    return error;
+}
+float train_network_sgd_gpu(network net, data d, int n)
+{
+    int batch = net.batch;
+    float *X = calloc(batch*d.X.cols, sizeof(float));
+    float *y = calloc(batch*d.y.cols, sizeof(float));
+
+    int i;
+    float sum = 0;
+    for(i = 0; i < n; ++i){
+        get_batch(d, batch, X, y);
+        float err = train_network_datum_gpu(net, X, y);
+        sum += err;
+    }
+    free(X);
+    free(y);
+    return (float)sum/(n*batch);
+}
+#endif
+
+
 float train_network_datum(network net, float *x, float *y)
 {
     forward_network(net, x, y, 1);

src/network.h

@@ -30,8 +30,8 @@ typedef struct {
     float *output;
 
     #ifdef GPU
-    cl_mem input_cl;
-    cl_mem output_cl;
+    cl_mem *input_cl;
+    cl_mem *truth_cl;
     #endif
 } network;
 
@@ -41,6 +41,7 @@ void backward_network_gpu(network net, cl_mem input);
 void update_network_gpu(network net);
 cl_mem get_network_output_cl_layer(network net, int i);
 cl_mem get_network_delta_cl_layer(network net, int i);
+float train_network_sgd_gpu(network net, data d, int n);
 #endif
 
 network make_network(int n, int batch);