Algebra

To demonstrate greycat’s machine learning capabilities, let’s start by building a dataset with correlated outputs and inputs

    @library("algebra");
    use util;
    use compute;
    use ml;
    use nn;

    var inputs = 2;
    var outputs = 1;
    var setSize = 1000;
    var tensorType = TensorType::f64;

    var random: Random = Random::new();
    random.setSeed(42);

    var tensorInput = Tensor {};
    tensorInput.init(tensorType, [setSize, inputs]);
    var index = [0, 0]; // index needs to be an int array of size = tensor dimensions
    do {
        tensorInput.set(index, random.uniformf(0.1, 1.0));
    } while (tensorInput.incPos(index)); // loop over flattened tensor positions

    // regression or autoencoder output
    var tensorOutput = Tensor {};
    tensorOutput.init(tensorType, [setSize, outputs]);
    index = [0, 0]; // index needs to be an int array of size = tensor dimensions
    var corelationCoefficients = [0.2, 0.6]; // set the corelation that we would like to determine later
    do {
        var noise = random.uniformf(0.0, 1.0) * (1 - corelationCoefficients[0] - corelationCoefficients[1]); // add some noise so the data does not perfectly corelate
        var val = corelationCoefficients[0] * tensorInput.get([index[0], 0]) + corelationCoefficients[1] * tensorInput.get([index[0], 1]) + noise;
        tensorOutput.set(index, val);
    } while (tensorOutput.incPos(index));

    // classification 1 hot encoded output
    var classes = 3;
    var tensorClassOutput = Tensor {};
    tensorClassOutput.init(tensorType, [setSize, classes]);
    var c: int;
    for (var i = 0; i < setSize; i++) {
        c = random.uniform(0, classes - 1); // select random class
        for ( var j= 0; j < classes; j++){
            if(j==c){
                tensorClassOutput.set([i, j], 1.0);
            } else{
                tensorClassOutput.set([i, j], 0.0);
            }

        }
    }

GaussianND (Gaussian n-dimensional)

Let’s build a Gaussian Profile from the inputs and outputs, and then crop the profile to also get the input and output profiles separately, and finally, we can get the correlation between the inputs and outputs

    var combinedTensor = Tensor {};
    combinedTensor.init(tensorType, [setSize, inputs + outputs]);
    for (var i = 0; i < setSize; i++) {
        // copy the input and output tensors into a single tensor
        combinedTensor.copyElementsFrom(tensorInput, i * inputs, inputs, i * (inputs + outputs));
        combinedTensor.copyElementsFrom(tensorOutput, i * outputs, outputs, i * (inputs + outputs) + inputs);
    }

    var profile = GaussianND {};
    // Learn the profile from the combined tensor
    profile.learn(combinedTensor);

    // Will return a table showing the correlation between inputs + outputs where
    // Col:0 = input-0, Col:1 = input-1, Col:2 = output-0
    // Row:0 = input-0, Row:1 = input-1, Row:2 = output-0
    var correlationTable = profile.correlation().toTable();
    println(correlationTable);


    // Crop the profile to get the input profiles
    var inputProfile = profile.crop(0, inputs - 1);

    // Crop the profile to get the output profiles
    var outputProfile = profile.crop(inputs, inputs + outputs - 1);

Printing the correlationTablem you will able to see the correlation information in the data attribute.

{"_type":"core.Table","meta":[{"_type":"core.TableColumnMeta","type":"core.float","size":3,"min":0.0518855985,"max":1.0,"avg":0.4624200185,"std":0.4866576421,"index":false,"header":null},{"_type":"core.TableColumnMeta","type":"core.float","size":3,"min":0.0518855985,"max":1.0,"avg":0.6509063759,"std":0.5211313808,"index":false,"header":null},{"_type":"core.TableColumnMeta","type":"core.float","size":3,"min":0.3353744572,"max":1.0,"avg":0.7454026622,"std":0.3585398777,"index":false,"header":null}],"data":[[1.0,0.0518855985,0.3353744572],[0.0518855985,1.0,0.9008335293],[0.3353744572,0.9008335293,1.0]]}

PCA

PCA is a dimensionality reduction technique that can be used to reduce the number of dimensions in a dataset while retaining most of the variance in the data. Requires a learned GaussianND to utilize, and then we can set the number of dimensions we want to keep.

    var pca = PCA {};
    pca.learn(inputProfile.correlation(), inputProfile.avg(), inputProfile.std(), null);
    pca.set_dim(inputs); 

Regression

Leverage the previously defined Tensor and PCA to start building a regression network that can predict the outputs from the inputs.

    var nnRegression: RegressionNetwork = RegressionNetwork::new(inputs, outputs, tensorType, false, null, 1234);

    // Set the pre and post processes for the network to use PCA and standard scaling respectively
    nnRegression.setPreProcess(PreProcessType::pca_scaling, pca);
    nnRegression.setPostProcess(PostProcessType::standard_scaling, outputProfile);

    nnRegression.addDenseLayer(5, true, ComputeActivationRelu {}, null);
    nnRegression.addDenseLayer(outputs, true, ComputeActivationRelu {}, null);

    nnRegression.setLoss(ComputeRegressionLoss::square, ComputeReduction::auto);
    nnRegression.setOptimizer(ComputeOptimizerAdam {}); 

    var res: ComputeModel = nnRegression.build(true);
    var engine: ComputeEngine = ComputeEngine::new();

    // Initialize the network with a batch size of 100 
    var batchSize = 100;
    nnRegression.initWithBatch(res, engine, null, batchSize);
    var nnInputs = nnRegression.getInput(engine);
    var nnTargets = nnRegression.getTarget(engine);

    // Split the dataset into training and testing sets (80% training, 20% testing) 
    var trainSetSize = (0.8 * setSize) as int;
    var testSetSize = (0.2 * setSize) as int;

    // saving the best model state
    var bestState = ComputeState {};
    var bestLoss : float?;
    
    for (var epoch = 0; epoch < 10; epoch++) {
        // track the loss for each epoch
        var loss : Tensor;
        var trainLossSum : float = 0.0;
        var testLossSum : float = 0.0;
        var trainIterations = trainSetSize / batchSize;
        // train
        for (var i = 0; i < trainIterations; i++) {
            nnInputs.copyElementsFrom(tensorInput, i * batchSize, batchSize, 0);
            nnTargets.copyElementsFrom(tensorOutput, i * batchSize, batchSize, 0);
            nnRegression.miniBatch(engine); // accumulate gradients, optimize after loop
            loss = nnRegression.getDisplayLoss(engine); 
            trainLossSum = loss.sum();
        }
        nnRegression.optimize(engine); // optimize with accumulated gradients

        // test
        var testIterations = testSetSize / batchSize;
        for (var i = 0; i < testIterations; i++) {
            nnInputs.copyElementsFrom(tensorInput, i * batchSize + trainSetSize, batchSize, 0);
            nnTargets.copyElementsFrom(tensorOutput, i * batchSize + trainSetSize, batchSize, 0);
            nnRegression.test(engine);
            loss = nnRegression.getDisplayLoss(engine);
            testLossSum = loss.sum();
        }println("Epoch ${epoch} loss train: ${trainLossSum} test: ${testLossSum}");
        // saving model state if first epoch or loss reduced
        if (bestLoss == null ||  testLossSum < bestLoss){
            engine.saveState(bestState);
            bestLoss = testLossSum;
        }
    }

    // loading model state
    engine.configure(true); // set to inferecne mode (only forward)
    engine.loadState(bestState);
    
    // inferece using latest test batch
    var tensorOut = nnRegression.predict(engine, nnInputs);
    var arrayOut = [];
    for (var col = 0; col < tensorOut.shape()[1]; col++) {
        arrayOut.add(tensorOut.get([0, col]) as float);
    }
    println("Inference output: ${arrayOut}");
  

Classification

    var nnClassification = ClassificationNetwork::new(inputs, classes, tensorType, false, null, 1234, true, true, false );

    // Set the pre process for the network to use PCA scaling
    nnClassification.setPreProcess(PreProcessType::pca_scaling, pca);

    nnClassification.addDenseLayer(5, true, ComputeActivationRelu {}, null);
    nnClassification.addDenseLayer(classes, true, ComputeActivationRelu {}, null);

    nnClassification.setLoss(ComputeClassificationLoss::categorical_cross_entropy, ComputeReduction::auto);
    nnClassification.setOptimizer(ComputeOptimizerAdam {}); 

    var res: ComputeModel = nnClassification.build(true);
    var engine: ComputeEngine = ComputeEngine::new();

    // Initialize the network with a batch size of 100 
    var batchSize = 100;
    nnClassification.initWithBatch(res, engine, null, batchSize);
    var nnInputs = nnClassification.getInput(engine);
    var nnTargets = nnClassification.getTarget(engine);

    println(nnTargets.type());
    println(tensorClassOutput.type());

    // Split the dataset into training and testing sets (80% training, 20% testing) 
    var trainSetSize = (0.8 * setSize) as int;
    var testSetSize = (0.2 * setSize) as int;

    // saving the best model state
    var bestState = ComputeState {};
    var bestLoss : float?;

    for (var epoch = 0; epoch < 10; epoch++) {
        // track the loss for each epoch
        var loss : Tensor;
        var trainLossSum : float = 0.0;
        var testLossSum : float = 0.0;
        var trainIterations = trainSetSize / batchSize;
        // train
        for (var i = 0; i < trainIterations; i++) {
            nnInputs.copyElementsFrom(tensorInput, i * batchSize, batchSize, 0);
            nnTargets.copyElementsFrom(tensorClassOutput, i * batchSize, batchSize, 0);
            loss = nnClassification.miniBatch(engine); // accumulate gradients, optimize after loop
            trainLossSum = trainLossSum + loss.sum();
        }
        nnClassification.optimize(engine); // optimize with accumulated gradients

        // test
        var testIterations = testSetSize / batchSize;
        for (var i = 0; i < testIterations; i++) {
            nnInputs.copyElementsFrom(tensorInput, i * batchSize + trainSetSize, batchSize, 0);
            nnTargets.copyElementsFrom(tensorOutput, i * batchSize + trainSetSize, batchSize, 0);
            loss = nnClassification.test(engine);
            testLossSum = testLossSum + loss.sum();
        }
        println("Epoch ${epoch} loss train: ${trainLossSum} test: ${testLossSum}");
        // saving model state if first epoch or loss reduced
        if (bestLoss == null ||  testLossSum < bestLoss){
            engine.saveState(bestState);
            bestLoss = testLossSum;
        }
    }

    // loading model state
    engine.configure(true); // set to inferecne mode (only forward)
    engine.loadState(bestState);
    
    // inferece using latest test batch
    var tensorOut = nnClassification.predict(engine, nnInputs);
    var arrayOut = [];
    for (var col = 0; col < tensorOut.shape()[1]; col++) {
        arrayOut.add(tensorOut.get([0, col]) as float);
    }
    println("Inference output: ${arrayOut}");

Auto encoder

    var nnAutoEnCoder = AutoEncoderNetwork::new(inputs, tensorType, false, null, 1234);

    // Set the pre and post processes for the network to use PCA and standard scaling respectively
    nnAutoEnCoder.setPreProcess(PreProcessType::pca_scaling, pca);
    nnAutoEnCoder.setPostProcess(PostProcessType::standard_scaling, inputProfile);

    nnAutoEnCoder.addDenseLayer(5, true, ComputeActivationRelu {}, null);
    nnAutoEnCoder.addLinearLayer(inputs, true, null);

    nnAutoEnCoder.setLoss(ComputeRegressionLoss::square, ComputeReduction::auto);
    nnAutoEnCoder.setOptimizer(ComputeOptimizerAdam {}); 

    var res: ComputeModel = nnAutoEnCoder.build(true);
    var engine: ComputeEngine = ComputeEngine::new();

    // Initialize the network with a batch size of 100 
    var batchSize = 100;
    nnAutoEnCoder.initWithBatch(res, engine, null, batchSize);
    var nnInputs = nnAutoEnCoder.getInput(engine);
    var nnTargets = nnAutoEnCoder.getTarget(engine);

    // Split the dataset into training and testing sets (80% training, 20% testing) 
    var trainSetSize = (0.8 * setSize) as int;
    var testSetSize = (0.2 * setSize) as int;

    // saving the best model state
    var bestState = ComputeState {};
    var bestLoss : float?;

    // train loop
    for (var epoch = 0; epoch < 10; epoch++) {
        // track the loss for each epoch
        var loss : Tensor;
        var trainLossSum : float = 0.0;
        var testLossSum : float = 0.0;
        var trainIterations = trainSetSize / batchSize;
        // train
        for (var i = 0; i < trainIterations; i++) {
            nnInputs.copyElementsFrom(tensorInput, i * batchSize, batchSize, 0);
            nnTargets.copyElementsFrom(tensorInput, i * batchSize, batchSize, 0); // target same as inputs
            nnAutoEnCoder.miniBatch(engine); // accumulate gradients, optimize after loop
            loss = nnAutoEnCoder.getDisplayLoss(engine); 
            trainLossSum = loss.sum();
        }
        nnAutoEnCoder.optimize(engine); // optimize with accumulated gradients

        // test
        var testIterations = testSetSize / batchSize;
        for (var i = 0; i < testIterations; i++) {
            nnInputs.copyElementsFrom(tensorInput, i * batchSize + trainSetSize, batchSize, 0);
            nnTargets.copyElementsFrom(tensorOutput, i * batchSize + trainSetSize, batchSize, 0);
            nnAutoEnCoder.test(engine);
            loss = nnAutoEnCoder.getDisplayLoss(engine);
            testLossSum = loss.sum();
        }
        println("Epoch ${epoch} loss train: ${trainLossSum} test: ${testLossSum}");

        // saving model state if first epoch or loss reduced
        if (bestLoss == null ||  testLossSum < bestLoss){
            engine.saveState(bestState);
            bestLoss = testLossSum;
        }
    }

    // loading model state
    engine.configure(true); // set to inferecne mode (only forward)
    engine.loadState(bestState);
    
    // inferece using latest test batch
    var tensorEndoded = nnAutoEnCoder.encode(engine, nnInputs);
    println("Shape of encoded tensor determined by hidden layer size: ${tensorEndoded.shape()}");
    var tensorOut = nnAutoEnCoder.decode(engine,tensorEndoded);
    var arrayOut = [];
    for (var col = 0; col < tensorOut.shape()[1]; col++) {
        arrayOut.add(tensorOut.get([0, col]) as float);
    }
    println("Decoded output: ${arrayOut}");