added regression

IVOES · Jan 6, 2017 · 65bc62a · 65bc62a
1 parent 45d7c92
commit 65bc62a
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -2,7  2,7 @@
 A portable, header-only, neural network framework written in C
 
 ## Features
-* Sidmoid, ReLU, tanh, Softmax activations
 * Sidmoid, ReLU, tanh, Softmax, Linear activations
 * Cross-entropy loss, Mean squared error
 * Batch, SGD, Mini-Batch
 * L2 Regularization

diff --git a/src/function.h b/src/function.h
@@ -39,6  39,9 @@ void tanH(Matrix* input);
 // applies softmax function to each row of $input
 void softmax(Matrix* input);
 
 // applies linear function to each row of $input
 void linear(Matrix* input);
 
 // sample from the unit guassian distribution (mean = 0, variance = 1)
 float box_muller();
 
@@ -125,6  128,9 @@ void softmax(Matrix* input){
     }
 }
 
 // operates on each row
 void linear(Matrix* input){}
 
 // adapted from wikipedia
 float box_muller(){
     const float epsilon = FLT_MIN;

diff --git a/src/network.h b/src/network.h
@@ -27,8  27,14 @@ void forwardPass(Network* network, Matrix* input);
 
 // calculate the cross entropy loss between two datasets with 
 // optional regularization (must provide network if using regularization)
 // [normal cross entropy]   1/2(regStrength)[normal l2 reg]
 float crossEntropyLoss(Network* network, Matrix* prediction, Matrix* actual, float regularizationStrength);
 
 // calculate the mean squared error between two datasets with 
 // optional regularization (must provide network if using regularization)
 // 1/2[normal mse]   1/2(regStrength)[normal l2 reg]
 float meanSquaredError(Network* network, Matrix* prediction, Matrix* actual, float regularizationStrength);
 
 // returns indices corresponding to highest-probability classes for each
 // example previously inputted
 // assumes final output is in the output layer of network
@@ -128,6  134,35 @@ float crossEntropyLoss(Network* network, Matrix* prediction, Matrix* actual, flo
     return ((-1.0 / actual->rows) * total_err)   (regularizationStrength * .5 * reg_err);
 }
 
 // matrixes of size [num examples] x [num classes]
 float meanSquaredError(Network* network, Matrix* prediction, Matrix* actual, float regularizationStrength){
     assert(prediction->rows == actual->rows);
     assert(prediction->cols == actual->cols);
     float total_err = 0;
     int i, j, k;
     for (i = 0; i < prediction->rows; i  ){
         float cur_err = 0;
         for (j = 0; j < prediction->cols; j  ){
             float tmp = actual->data[i][j] - prediction->data[i][j];
             cur_err  = tmp * tmp;
         }
         total_err  = cur_err;
     }
     float reg_err = 0;
     if (network != NULL){
         for (i = 0; i < network->numConnections; i  ){
             Matrix* weights = network->connections[i]->weights;
             for (j = 0; j < weights->rows; j  ){
                 for (k = 0; k < weights->cols; k  ){
                     reg_err  = weights->data[j][k] * weights->data[j][k];
                 }
             }
         }
     }
     return ((0.5 / actual->rows) * total_err)   (regularizationStrength * .5 * reg_err);
 }
 
 
 int* predict(Network* network){
     int i, j, max;
     Layer* outputLayer = network->layers[network->numLayers - 1];

diff --git a/src/optimizer.h b/src/optimizer.h
@@ -117,8  117,13 @@ void batchGradientDescent(Network* network, Matrix* data, Matrix* classes, LOSS_
                     if (layer == network->numLayers - 1){
                         // calculate output layer's error
                         copyValuesInto(to->input, errori[layer]);
-                        for (j = 0; j < errori[layer]->cols; j  ){
-                            errori[layer]->data[0][j] -= target->data[0][j];
                         if (lossFunction == CROSS_ENTROPY_LOSS){
                             for (j = 0; j < errori[layer]->cols; j  ){
                                 errori[layer]->data[0][j] -= target->data[0][j];
                             }
                         }
                         else{
                             errori[layer]->data[0][0] -= target->data[0][0];
                         }
 
                         // calculate dWi and dbi
@@ -227,12  232,25 @@ void batchGradientDescent(Network* network, Matrix* data, Matrix* classes, LOSS_
 
             // if verbose is set, print loss every 100 epochs
             if (verbose != 0){
-                if (epoch % 100 == 0){
                 if (epoch % 100 == 0 || epoch == 1){
                     forwardPass(network, data);
-                    printf("EPOCH %d: loss is %f\n", epoch, crossEntropyLoss(network, network->layers[network->numLayers - 1]->input, classes, regularizationStrength));
                     if (lossFunction == CROSS_ENTROPY_LOSS){
                         printf("EPOCH %d: loss is %f\n", epoch, crossEntropyLoss(network, network->layers[network->numLayers - 1]->input, classes, regularizationStrength));
                     }
                     else{
                         printf("EPOCH %d: loss is %f\n", epoch, meanSquaredError(network, network->layers[network->numLayers - 1]->input, classes, regularizationStrength));
                     }
                 }
             }
         }
 
         // free batch data for next batch creation
         for (i = 0; i < numBatches; i  ){
             free(dataBatches[i]);
             free(classBatches[i]);
         }
         free(dataBatches);
         free(classBatches);
     }
 
     // free all reusable matrices
@@ -264,13  282,6 @@ void batchGradientDescent(Network* network, Matrix* data, Matrix* classes, LOSS_
         free(fprimei);
         free(inputTi);
     }
-
-    for (i = 0; i < numBatches; i  ){
-        free(dataBatches[i]);
-        free(classBatches[i]);
-    }
-    free(dataBatches);
-    free(classBatches);
 }
 
 #endif
diff --git a/tests/optimizer_tests.c b/tests/optimizer_tests.c
@@ -103,13  103,46 @@ int main(){
     printf("Starting accuracy of %f\n", accuracy(networkF, trainingDataF, trainingClassesF));
     batchGradientDescent(networkF, trainingDataF, trainingClassesF, CROSS_ENTROPY_LOSS, 20, .01, 0, .01, .5, 1000, 1, 1);
     printf("Final accuracy of %f\n", accuracy(networkF, trainingDataF, trainingClassesF));
 
     // test on regression on y=x^2   15
     float** dataReg = (float**)malloc(sizeof(float*) * 1000);
     for (i = 0; i < 1000; i  ){
         dataReg[i] = (float*)malloc(sizeof(float) * 1);
         dataReg[i][0] = rand() % 20;
     }
     Matrix* trainingDataReg = createMatrix(1000, 1, dataReg);
     float** classesReg = (float**)malloc(sizeof(float*) * 1000);
     for (i = 0; i < 1000; i  ){
         classesReg[i] = (float*)malloc(sizeof(float) * 1);
         classesReg[i][0] = dataReg[i][0] * dataReg[i][0]   15;
     }
     Matrix* trainingClassesReg = createMatrix(1000, 1, classesReg);
 
     int hiddenSizeReg[] = {20};
     void (*hiddenActivationsReg[])(Matrix*) = {relu};
     Network* networkReg = createNetwork(1, 1, hiddenSizeReg, hiddenActivationsReg, 1, linear);
 
     printf("\nTESTING ON PARABOLA REGRESSION:\n");
     float** oneEx = (float**)malloc(sizeof(float*));
     oneEx[0] = (float*)malloc(sizeof(float));
     oneEx[0][0] = 5.0;
     Matrix* oneExData = createMatrix(1, 1, oneEx);
     forwardPass(networkReg, oneExData);
     printf("Initially maps 5 to %f but should map to 40\n", networkReg->layers[networkReg->numLayers - 1]->input->data[0][0]);
     batchGradientDescent(networkReg, trainingDataReg, trainingClassesReg, MEAN_SQUARED_ERROR, 20, .01, 0, .001, .9, 500, 1, 1);
     forwardPass(networkReg, oneExData);
     printf("After training maps 5 to %f but should map to 40\n", networkReg->layers[networkReg->numLayers - 1]->input->data[0][0]);
 
     destroyMatrix(trainingData);
     destroyMatrix(trainingClasses);
     destroyNetwork(network2);
     destroyMatrix(trainingDataF);
     destroyMatrix(trainingClassesF);
     destroyNetwork(networkF);
     destroyMatrix(oneExData);
     destroyMatrix(trainingDataReg);
     destroyMatrix(trainingClassesReg);
     destroyNetwork(networkReg);
 
     return 0;
 }