We Keep Coding

My neural networks works well only with no hidden layers

You may think I am crazy, but I decided to write a neural network from scratch in C# for studying purposes. Please, be patient, I still have little experience)). English is not my first language, so I am sorry for it in advance.
I started with a program for handwritten digit recognizing with the MNIST database. I've read through a book about the algorithms inside the process and wrote this code.
public class NeuralNetwork
{
private List<Matrix<double>> weights = new List<Matrix<double>>();
private List<Vector<double>> biases = new List<Vector<double>>();
private Random random = new Random();
private List<Image> test_data;
private int[] layer_sizes;
public NeuralNetwork(params int[] layers)
{
layer_sizes = layers;
for (int i = 0; i < layers.Length - 1; i++)
{
var weigthLayer = Matrix<double>.Build.Dense(layers[i + 1], layers[i], (k, j) => random.NextDouble());
weights.Add(weigthLayer);
}
for (int i = 1; i < layers.Length; i++)
{
var biasesLayer = Vector<double>.Build.Dense(layers[i], (k) => random.NextDouble());
biases.Add(biasesLayer);
}
}
public Vector<double> FeedForward(Vector<double> a)
{
for (int i = 0; i < weights.Count; i++)
{
a = Sigmoid(weights[i].Multiply(a) + biases[i]);
}
return Sigmoid(a);
}
public void SGD(ITrainingDataProvider dataProvider, int epochs, int chuck_size, double eta)
{
test_data = new MnistReader().ReadTestData();
Console.WriteLine("SGD algorithm started");
var training_data = dataProvider.ReadTrainingData();
Console.WriteLine("Training data has beeen read");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
for (int epoch = 0; epoch < epochs; epoch++)
{
training_data = training_data.OrderBy(item => random.Next()).ToList();
List<List<Image>> chunks = training_data.ChunkBy(chuck_size);
foreach (List<Image> chunk in chunks)
{
ProcessChunk(chunk, eta);
}
Console.WriteLine($"Epoch: {epoch + 1}/{epochs}");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
}
Console.WriteLine("Done!");
}
private double Test(List<Image> data)
{
int count = 0;
foreach (Image im in data)
{
var output = FeedForward(im.DataToVector());
int number = output.MaximumIndex();
if (number == (int)im.Label)
{
count++;
}
}
return (double)count / data.Count * 100;
}
private void ProcessChunk(List<Image> chunk, double eta)
{
Delta[] deltas = new Delta[chunk.Count];
for (int i = 0; i < chunk.Count; i++)
{
Image image = chunk[i];
var input = image.DataToVector();
var desired_output = Vector<double>.Build.Dense(layer_sizes[layer_sizes.Length - 1]);
desired_output[(int)image.Label] = 1;
Delta delta = BackPropagation(input, desired_output);
deltas[i] = delta;
}
Delta sum = deltas[0];
for (int i = 1; i < deltas.Length; i++)
{
sum += deltas[i];
}
Delta average_delta = sum / deltas.Length;
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
weights[i] += average_delta.d_weights[i].Multiply(eta);
biases[i] += average_delta.d_biases[i].Multiply(eta);
}
}
private Delta BackPropagation(Vector<double> input, Vector<double> desired_output)
{
List<Vector<double>> activations = new List<Vector<double>>();
List<Vector<double>> zs = new List<Vector<double>>();
Vector<double> a = input;
activations.Add(input);
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
var z = weights[i].Multiply(a) + biases[i];
zs.Add(z);
a = Sigmoid(z);
activations.Add(a);
}
List<Vector<double>> errors = new List<Vector<double>>();
List<Matrix<double>> delta_weights = new List<Matrix<double>>();
List<Vector<double>> delta_biases = new List<Vector<double>>();
var error = CDerivative(activations[activations.Count - 1], desired_output).HProd(SigmoidDerivative(zs[^1]));
errors.Add(error);
int steps = 0;
for (int i = layer_sizes.Length - 2; i >= 1; i--)
{
var layer_error = weights[i].Transpose().Multiply(errors[steps]).HProd(SigmoidDerivative(zs[i - 1]));
errors.Add(layer_error);
steps++;
}
errors.Reverse();
for (int i = layer_sizes.Length - 1; i >= 1; i--)
{
var delta_layer_weights = (errors[i - 1].ToColumnMatrix() * activations[i - 1].ToRowMatrix()).Multiply(-1);
delta_weights.Add(delta_layer_weights);
var delta_layer_biases = errors[i - 1].Multiply(-1);
delta_biases.Add(delta_layer_biases);
}
delta_biases.Reverse();
delta_weights.Reverse();
return new Delta { d_weights = delta_weights, d_biases = delta_biases };
}
private Vector<double> CDerivative(Vector<double> x, Vector<double> y)
{
return x - y;
}
private Vector<double> Sigmoid(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = 1.0 / (1.0 + Math.Exp(-x[i]));
}
return x;
}
private Vector<double> SigmoidDerivative(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = Math.Exp(-x[i]) / Math.Pow(1.0 + Math.Exp(-x[i]), 2);
}
return x;
}
}
Delta class. A simple DTO to store weights and biases changes in a single object.
public class Delta
{
public List<Matrix<double>> d_weights { get; set; }
public List<Vector<double>> d_biases { get; set; }
public static Delta operator +(Delta d1, Delta d2)
{
Delta result = d1;
for (int i = 0; i < d2.d_weights.Count; i++)
{
result.d_weights[i] += d2.d_weights[i];
}
for (int i = 0; i < d2.d_biases.Count; i++)
{
result.d_biases[i] += d2.d_biases[i];
}
return result;
}
public static Delta operator /(Delta d1, double d)
{
Delta result = d1;
for (int i = 0; i < d1.d_weights.Count; i++)
{
result.d_weights[i] /= d;
}
for (int i = 0; i < d1.d_biases.Count; i++)
{
result.d_biases[i] /= d;
}
return result;
}
}
Everything ended up working fine, however complex networks with 1 or more hidden layers don't show any significant results. They are getting best 70% accuracy and then the learning curve drops. The accuracy returns to its 20-30%. Typically, the graph looks like a square root function, but in my case it is more like a turned around quadratic parabola the graph of my tries with different amounts of neurons in the first hidden layer
After a few tries, I found out, that without any hidden layers the algorithm works just fine. It learns up to 90% of accuracy and then the graph never falls down. Apparently, the bug is somewhere in back-propagation algorithm. It doesn't cause any problems with only input and output layers, but it does, when I add a hidden layer.
I have been trying to find the problem for a long time and I hope that someone, smarter than me, will be able to help.
Thanks in advance!

Why is My Base 10 LSD Radix-Sort Faster Than My Bit-Shifted Radix Sort?

So I was experimenting with different kinds of Radix sort for a school project where we have to sort 500,000 random integers (that I generate myself, with a bounds between 0 to MaxValue for each integer) as quickly as possible. I first made a base 10 LSD (Least Significant Digit) Radix sort that averages around 110 to 115ms for sorting 500,000 random integers. Here's the code for it:
public static int[] RadixSort(int[] RandomNumbers)
{
List<int>[] Buckets = new List<int>[10];
int singleDigit = 0;
int[] temp;
int[] mult = new int[10] {1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000};
for (int z = 0; z < 10; z++)
{
Buckets[0] = new List<int>();
Buckets[1] = new List<int>();
Buckets[2] = new List<int>();
Buckets[3] = new List<int>();
Buckets[4] = new List<int>();
Buckets[5] = new List<int>();
Buckets[6] = new List<int>();
Buckets[7] = new List<int>();
Buckets[8] = new List<int>();
Buckets[9] = new List<int>();
if (z == 0)
{
temp = (int[])RandomNumbers.Clone();
}
else
{
temp = (int[])RandomNumbers.Clone();
for (int i = 0; i < SIZE; i++)
{
temp[i] /= (mult[z]);
}
}
for (int i = 0; i < SIZE; i++)
{
singleDigit = temp[i] % 10;
Buckets[singleDigit].Add(RandomNumbers[i]);
}
List<int> NewList = new List<int>(SIZE);
for (int i = 0; i < 10; i++)
{
NewList.AddRange(Buckets[i]);
}
int[] NewArray = NewList.ToArray();
RandomNumbers = NewArray;
}
return RandomNumbers;
}
However, I heard that using binary in a bit-shift Radix sort was faster. So I created a mask-based bit-shift Radix sort that overall looks less cluttered and has less operations going on within it, but its average sort speed is around 250ms. Here's the code for it:
public static int[] BitShiftRadixSort(int[] RandomNumbers)
{
List<int>[] Buckets = new List<int>[2];
int binary;
int mask;
for (int shift = 0; shift < 32; shift++)
{
Buckets[0] = new List<int>(SIZE);
Buckets[1] = new List<int>(SIZE);
mask = 1 << shift;
for (int i = 0; i < SIZE; i++)
{
binary = RandomNumbers[i] & mask;
if (binary != 0)
{
Buckets[1].Add(RandomNumbers[i]);
}
else
{
Buckets[0].Add(RandomNumbers[i]);
}
}
List<int> NewList = new List<int>(SIZE);
for (int i = 0; i < 2; i++)
{
NewList.AddRange(Buckets[i]);
}
int[] NewArray = NewList.ToArray();
RandomNumbers = NewArray;
}
return RandomNumbers;
}
I was expecting the bit-shift to be faster than the LSD Radix sort but that doesn't appear to be the case. Are mathematical operations in C# heavily optimized? I'll appreciate everyone's input!

Using Matt Timmermans suggestion of using base 256, this version for unsigned integers takes about 10ms for 500,000 32 bit unsigned integers. For signed integers, you would need to complement the sign bit and treat them as unsigned integers (then complement the sign back back).
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
namespace RadixSort
{
class Program
{
static void RadixSort(uint [] a, uint count)
{
uint [,] mIndex = new uint [4,256]; // count / index matrix
uint [] b = new uint [count]; // allocate temp array
uint i,j,m,n;
uint u;
for(i = 0; i < count; i++){ // generate histograms
u = a[i];
for(j = 0; j < 4; j++){
mIndex[j,(u & 0xff)]++;
u >>= 8;
}
}
for(j = 0; j < 4; j++){ // convert to indices
m = 0;
for(i = 0; i < 256; i++){
n = mIndex[j,i];
mIndex[j,i] = m;
m += n;
}
}
for(j = 0; j < 4; j++){ // radix sort
for(i = 0; i < count; i++){ // sort by current lsb
u = a[i];
m = (u>>((int)(j<<3)))&0xff;
b[mIndex[j,m]++] = u;
}
uint [] t = a; // swap references
a = b;
b = t;
}
}
static void Main(string[] args)
{
const int SIZE = 500000;
uint [] a = new uint[SIZE];
uint u;
Random r = new Random(1);
Stopwatch sw = new Stopwatch();
for (uint i = 0; i < a.Length; i++)
{
u = (uint)r.Next(1 << 16);
u = (u << 16) | (uint)r.Next(1 << 16);
a[i] = u;
}
sw.Start();
RadixSort(a, (uint)a.Length);
sw.Stop();
for (uint i = 1; i < a.Length; i++)
{
if(a[i] < a[i-1])
{
Console.WriteLine("failed");
break;
}
}
Console.WriteLine("milliseconds: {0:D}\n",sw.ElapsedMilliseconds);
}
}
}

Trying to perform 2D Cross correlation on 2 images without using Matlab/Numpy/etc

I am trying to perform Cross Correlation on these 2 images in C#:
Image, Template
Matlab says the result is supposed to look like: Matlab result
But this is my result: My result
Here is my Cross Correlation function:
public static Signal2D CrossCorrelation2D(Signal2D signal, Signal2D pulse) {
return InverseFFT2D(FFT2D(signal) * FFT2D(pulse).GetConjugate());
}
Here is my FFT2D:
public static Signal2D FFT2D(Signal2D signal) {
Signal2D result = new Signal2D(signal.Height, signal.Width);
for (int i = 0; i < result.Height; i++)
result[i] = new ComplexNumber[signal[i].Length];
//rows
for (int n = 0; n < signal.Height; n++) {
result[n] = FFT(signal[n]);
}
//columns
for (int i = 0; i < signal[0].Length; i++) {
ComplexNumber[] col = new ComplexNumber[signal.Height];
for (int j = 0; j < col.Length; j++) {
col[j] = result[j][i];
}
col = FFT(col);
for (int j = 0; j < col.Length; j++) {
result[j][i] = col[j];
}
}
return result;
}
Here is my FFT:
public static Signal FFT(Signal signal) {
int N = signal.Length;
if (N == 1)
return signal;
if ((N & (N - 1)) != 0)
throw new ArgumentOutOfRangeException("signal length must be a power of 2");
Signal evenArr = new Signal(N / 2);
Signal oddArr = new Signal(N / 2);
for (int i = 0; i < N / 2; i++) {
evenArr[i] = signal[2 * i];
}
evenArr = FFT(evenArr);
for (int i = 0; i < N / 2; i++) {
oddArr[i] = signal[2 * i + 1];
}
oddArr = FFT(oddArr);
Signal result = new Signal(N);
for (int k = 0; k < N / 2; k++) {
double w = -2.0 * k * Math.PI / N;
ComplexNumber wk = new ComplexNumber(Math.Cos(w), Math.Sin(w));
ComplexNumber even = evenArr[k];
ComplexNumber odd = oddArr[k];
result[k] = even + (wk * odd);
result[k + N / 2] = even - (wk * odd);
}
return result;
}
Here is my Signal multiplication (using pointwise multiplication):
public static Signal2D operator* (Signal2D a, Signal2D b) {
if (a.Height != b.Height || a.Width != b.Width)
throw new ArgumentException("Sizes must be equal");
Signal2D result = new Signal2D(a.Height, a.Width);
for (int y = 0; y < a.Height; y++) {
for (int x = 0; x < a.Width; x++) {
result[y][x] = a[y][x] * b[y][x];
}
}
return result;
}
Any help is appreciated, thanks.
Edit: I left the matlab image at the original size of 1023 by 1023 and overlayed my result. It looks like I may already be at the result, I am just not sure how Matlab pads the image. Overlayed results (The red is the white part from my result, the grey is the black part from my result. Black/white is from Matlab)

How to invert double[,] in C#

I am making a program, where I need to calculate linear regression, but I got stuck at inversion of a matrix.
I have
double[,] location = new double[3,3];
It was then filled with numbers, but then I do not know, how to count the inverse matrix for it like in Linear algebra.
I searched for a solution on the internet, but there was some Matrix class that I dont know how to convert my double[,] to.
So, do you know some elegant way to inverse double[,] like the inversion of matrixes in Linear algebra?

Here you have a working example, just copy the entire code into a console project and run it.
I took it from this link https://jamesmccaffrey.wordpress.com/2015/03/06/inverting-a-matrix-using-c/
using System;
using System.Collections.Generic;
using System.Linq;
namespace matrixExample
{
class Program
{
static void Main(string[] args)
{
double[][] m = new double[][] { new double[] { 7, 2, 1 }, new double[] { 0, 3, -1 }, new double[] { -3, 4, 2 } };
double[][] inv = MatrixInverse(m);
//printing the inverse
for (int i = 0; i < 3; i++)
{
for (int j = 0; j < 3; j++)
Console.Write(Math.Round(inv[i][j], 1).ToString().PadLeft(5, ' ') + "|");
Console.WriteLine();
}
}
static double[][] MatrixCreate(int rows, int cols)
{
double[][] result = new double[rows][];
for (int i = 0; i < rows; ++i)
result[i] = new double[cols];
return result;
}
static double[][] MatrixIdentity(int n)
{
// return an n x n Identity matrix
double[][] result = MatrixCreate(n, n);
for (int i = 0; i < n; ++i)
result[i][i] = 1.0;
return result;
}
static double[][] MatrixProduct(double[][] matrixA, double[][] matrixB)
{
int aRows = matrixA.Length; int aCols = matrixA[0].Length;
int bRows = matrixB.Length; int bCols = matrixB[0].Length;
if (aCols != bRows)
throw new Exception("Non-conformable matrices in MatrixProduct");
double[][] result = MatrixCreate(aRows, bCols);
for (int i = 0; i < aRows; ++i) // each row of A
for (int j = 0; j < bCols; ++j) // each col of B
for (int k = 0; k < aCols; ++k) // could use k less-than bRows
result[i][j] += matrixA[i][k] * matrixB[k][j];
return result;
}
static double[][] MatrixInverse(double[][] matrix)
{
int n = matrix.Length;
double[][] result = MatrixDuplicate(matrix);
int[] perm;
int toggle;
double[][] lum = MatrixDecompose(matrix, out perm,
out toggle);
if (lum == null)
throw new Exception("Unable to compute inverse");
double[] b = new double[n];
for (int i = 0; i < n; ++i)
{
for (int j = 0; j < n; ++j)
{
if (i == perm[j])
b[j] = 1.0;
else
b[j] = 0.0;
}
double[] x = HelperSolve(lum, b);
for (int j = 0; j < n; ++j)
result[j][i] = x[j];
}
return result;
}
static double[][] MatrixDuplicate(double[][] matrix)
{
// allocates/creates a duplicate of a matrix.
double[][] result = MatrixCreate(matrix.Length, matrix[0].Length);
for (int i = 0; i < matrix.Length; ++i) // copy the values
for (int j = 0; j < matrix[i].Length; ++j)
result[i][j] = matrix[i][j];
return result;
}
static double[] HelperSolve(double[][] luMatrix, double[] b)
{
// before calling this helper, permute b using the perm array
// from MatrixDecompose that generated luMatrix
int n = luMatrix.Length;
double[] x = new double[n];
b.CopyTo(x, 0);
for (int i = 1; i < n; ++i)
{
double sum = x[i];
for (int j = 0; j < i; ++j)
sum -= luMatrix[i][j] * x[j];
x[i] = sum;
}
x[n - 1] /= luMatrix[n - 1][n - 1];
for (int i = n - 2; i >= 0; --i)
{
double sum = x[i];
for (int j = i + 1; j < n; ++j)
sum -= luMatrix[i][j] * x[j];
x[i] = sum / luMatrix[i][i];
}
return x;
}
static double[][] MatrixDecompose(double[][] matrix, out int[] perm, out int toggle)
{
// Doolittle LUP decomposition with partial pivoting.
// rerturns: result is L (with 1s on diagonal) and U;
// perm holds row permutations; toggle is +1 or -1 (even or odd)
int rows = matrix.Length;
int cols = matrix[0].Length; // assume square
if (rows != cols)
throw new Exception("Attempt to decompose a non-square m");
int n = rows; // convenience
double[][] result = MatrixDuplicate(matrix);
perm = new int[n]; // set up row permutation result
for (int i = 0; i < n; ++i) { perm[i] = i; }
toggle = 1; // toggle tracks row swaps.
// +1 -greater-than even, -1 -greater-than odd. used by MatrixDeterminant
for (int j = 0; j < n - 1; ++j) // each column
{
double colMax = Math.Abs(result[j][j]); // find largest val in col
int pRow = j;
//for (int i = j + 1; i less-than n; ++i)
//{
// if (result[i][j] greater-than colMax)
// {
// colMax = result[i][j];
// pRow = i;
// }
//}
// reader Matt V needed this:
for (int i = j + 1; i < n; ++i)
{
if (Math.Abs(result[i][j]) > colMax)
{
colMax = Math.Abs(result[i][j]);
pRow = i;
}
}
// Not sure if this approach is needed always, or not.
if (pRow != j) // if largest value not on pivot, swap rows
{
double[] rowPtr = result[pRow];
result[pRow] = result[j];
result[j] = rowPtr;
int tmp = perm[pRow]; // and swap perm info
perm[pRow] = perm[j];
perm[j] = tmp;
toggle = -toggle; // adjust the row-swap toggle
}
// --------------------------------------------------
// This part added later (not in original)
// and replaces the 'return null' below.
// if there is a 0 on the diagonal, find a good row
// from i = j+1 down that doesn't have
// a 0 in column j, and swap that good row with row j
// --------------------------------------------------
if (result[j][j] == 0.0)
{
// find a good row to swap
int goodRow = -1;
for (int row = j + 1; row < n; ++row)
{
if (result[row][j] != 0.0)
goodRow = row;
}
if (goodRow == -1)
throw new Exception("Cannot use Doolittle's method");
// swap rows so 0.0 no longer on diagonal
double[] rowPtr = result[goodRow];
result[goodRow] = result[j];
result[j] = rowPtr;
int tmp = perm[goodRow]; // and swap perm info
perm[goodRow] = perm[j];
perm[j] = tmp;
toggle = -toggle; // adjust the row-swap toggle
}
// --------------------------------------------------
// if diagonal after swap is zero . .
//if (Math.Abs(result[j][j]) less-than 1.0E-20)
// return null; // consider a throw
for (int i = j + 1; i < n; ++i)
{
result[i][j] /= result[j][j];
for (int k = j + 1; k < n; ++k)
{
result[i][k] -= result[i][j] * result[j][k];
}
}
} // main j column loop
return result;
}
}
}

I guess this is what you are looking for:
static double[][] MatrixInverse(double[][] matrix)
{
// assumes determinant is not 0
// that is, the matrix does have an inverse
int n = matrix.Length;
double[][] result = MatrixCreate(n, n); // make a copy of matrix
for (int i = 0; i < n; ++i)
for (int j = 0; j < n; ++j)
result[i][j] = matrix[i][j];
double[][] lum; // combined lower & upper
int[] perm;
int toggle;
toggle = MatrixDecompose(matrix, out lum, out perm);
double[] b = new double[n];
for (int i = 0; i < n; ++i)
{
for (int j = 0; j < n; ++j)
if (i == perm[j])
b[j] = 1.0;
else
b[j] = 0.0;
double[] x = Helper(lum, b); //
for (int j = 0; j < n; ++j)
result[j][i] = x[j];
}
return result;
}
See Test Run - Matrix Inversion Using C# for reference.

Neural Net - Feed Forward, Matrix Multiplication in C#

I'm attempting to make a Neural Network in C#, I based the design in a python code I made a while back. But somehow the end result is not the same.
I'm new to C# and I'm using it in Unity, so I have limitation to library uses.
In python numpy can do matrix multiplications with the numpy.dot() method. I Haven't found something similar in C#, especially in Unity. So I had to do it by hand.
The Python code:
import numpy as np
class NN:
def __init__(self, n_input, n_hidden_layers, n_hidden_nodes, n_output):
self.weights_hidden = []
for n in range(n_hidden_layers + 1):
if n == 0:
size = n_input, n_hidden_nodes
elif n == n_hidden_layers:
size = n_hidden_nodes, n_output
else:
size = n_hidden_nodes, n_hidden_nodes
self.weights_hidden.append(
np.random.random(size)
)
#staticmethod
def activation(x):
return np.tanh(x)
def feed_forward(self, ip):
input_values = (ip - np.mean(ip, axis=0)) / np.std(ip, axis=0)
for w, weights in enumerate(self.weights_hidden):
if w == 0:
result = input_values
result = np.array(
map(self.activation, result.dot(weights))
)
return result
ANN = NN(n_input=5, n_hidden_layers=2, n_hidden_nodes=3, n_output=1)
print ANN.feed_forward([1, 2, 3, 4, 5])
My attempt to convert it to C#.
using UnityEngine;
using System.Collections;
public class neural_net : MonoBehaviour {
int n_inputs;
int n_hidden_layers;
int n_hidden_nodes;
int n_outputs;
float[] inputs;
ArrayList hidden_weights;
ArrayList hidden_results;
float[] output_results;
public void init(int n_inputs, int n_hidden_layers, int n_hidden_nodes, int n_outputs){
this.n_inputs = n_inputs;
this.n_hidden_layers = n_hidden_layers;
this.n_hidden_nodes = n_hidden_nodes;
this.n_outputs = n_outputs;
this.hidden_weights = new ArrayList ();
this.hidden_results = new ArrayList ();
this.output_results = new float[n_outputs];
int rows;
int columns;
for (int h = 0; h < n_hidden_layers + 2; h++) {
if (h == 0){
// input -> hidden
rows = n_inputs;
columns = n_hidden_nodes;
}
else if(h == n_hidden_layers + 1){
// hidden -> output
rows = n_hidden_nodes;
columns = n_outputs;
}
else {
// hidden -> hidden
rows = n_hidden_nodes;
columns = n_hidden_nodes;
}
float[] hidden_result = new float[rows*columns];
hidden_results.Add(hidden_results);
float[,] target = new float[rows,columns];
string test = "";
for(int r = 0; r < rows; r++){
for(int c = 0; c < columns; c++){
target[r,c] = Random.Range(0.0f, 1.0f);
test += target[r,c] + ", ";
}
}
hidden_weights.Add(target);
}
}
float activation(float x){
// tanh(x);
return (1 - Mathf.Exp (-2 * x)) / (1 + Mathf.Exp (-2 * x));
}
float[] _dot_matrix(float[] results, float[,] weights){
float[] new_matrix = new float[weights.GetLength(1)];
string t0 = "";
for (int r = 0; r < weights.GetLength(1); r++){
float res = 0;
for (int c = 0; c < weights.GetLength(0); c++) {
res += results[c] * weights[c,r];
}
new_matrix[r] = res;
}
return new_matrix;
}
float[] _map_activation(float[] pre_results){
float[] results = new float[pre_results.Length];
for (int i = 0; i < results.Length; i++) {
results[i] = activation(pre_results[i]);
}
return results;
}
float[] feed_forward(){
int h;
for (h = 0; h < n_hidden_layers + 2; h++) {
float[] dot_matrix_result;
if(h == 0){
dot_matrix_result = _dot_matrix(inputs, (float[,])hidden_weights[h]);
}
else if (h == n_hidden_layers +1){
dot_matrix_result = _dot_matrix((float[])hidden_results[h-1], (float[,])hidden_weights[h]);
output_results = _map_activation(dot_matrix_result);
break;
}
else {
dot_matrix_result = _dot_matrix((float[])hidden_results[h-1], (float[,])hidden_weights[h]);
}
float[] result = _map_activation(dot_matrix_result);
hidden_results[h] = _map_activation(result);
}
return output_results;
}
float[] normalize_input(float[] inputs){
float sum = 0.0f;
for (int i = 0; i < inputs.Length; i++) {
sum += inputs[i] ;
}
float average = sum / inputs.Length;
float[] deviations = new float[inputs.Length];
for (int i = 0; i < inputs.Length; i++) {
deviations[i] = Mathf.Pow(inputs[i] - average,2);
}
float sum_deviation = 0;
for (int i = 0; i < deviations.Length; i++) {
sum_deviation += deviations[i];
}
float variance = sum_deviation / deviations.Length;
float std = Mathf.Sqrt (variance);
for (int i = 0; i < inputs.Length; i++) {
inputs[i] = (inputs[i] - average)/std;
}
return inputs;
}
public void start_net(float[] inputs){
this.inputs = normalize_input(inputs);
feed_forward ();
}
}
I run the net from other script using the init method and then the start_net() method.
I made a test with not random weights and fixed input data, but it didn't came to the same result as the python code.
What's wrong with the C# code?