I'm attempting to make a Neural Network in C#, I based the design in a python code I made a while back. But somehow the end result is not the same.
I'm new to C# and I'm using it in Unity, so I have limitation to library uses.
In python numpy can do matrix multiplications with the numpy.dot() method. I Haven't found something similar in C#, especially in Unity. So I had to do it by hand.
The Python code:
import numpy as np
class NN:
def __init__(self, n_input, n_hidden_layers, n_hidden_nodes, n_output):
self.weights_hidden = []
for n in range(n_hidden_layers + 1):
if n == 0:
size = n_input, n_hidden_nodes
elif n == n_hidden_layers:
size = n_hidden_nodes, n_output
else:
size = n_hidden_nodes, n_hidden_nodes
self.weights_hidden.append(
np.random.random(size)
)
#staticmethod
def activation(x):
return np.tanh(x)
def feed_forward(self, ip):
input_values = (ip - np.mean(ip, axis=0)) / np.std(ip, axis=0)
for w, weights in enumerate(self.weights_hidden):
if w == 0:
result = input_values
result = np.array(
map(self.activation, result.dot(weights))
)
return result
ANN = NN(n_input=5, n_hidden_layers=2, n_hidden_nodes=3, n_output=1)
print ANN.feed_forward([1, 2, 3, 4, 5])
My attempt to convert it to C#.
using UnityEngine;
using System.Collections;
public class neural_net : MonoBehaviour {
int n_inputs;
int n_hidden_layers;
int n_hidden_nodes;
int n_outputs;
float[] inputs;
ArrayList hidden_weights;
ArrayList hidden_results;
float[] output_results;
public void init(int n_inputs, int n_hidden_layers, int n_hidden_nodes, int n_outputs){
this.n_inputs = n_inputs;
this.n_hidden_layers = n_hidden_layers;
this.n_hidden_nodes = n_hidden_nodes;
this.n_outputs = n_outputs;
this.hidden_weights = new ArrayList ();
this.hidden_results = new ArrayList ();
this.output_results = new float[n_outputs];
int rows;
int columns;
for (int h = 0; h < n_hidden_layers + 2; h++) {
if (h == 0){
// input -> hidden
rows = n_inputs;
columns = n_hidden_nodes;
}
else if(h == n_hidden_layers + 1){
// hidden -> output
rows = n_hidden_nodes;
columns = n_outputs;
}
else {
// hidden -> hidden
rows = n_hidden_nodes;
columns = n_hidden_nodes;
}
float[] hidden_result = new float[rows*columns];
hidden_results.Add(hidden_results);
float[,] target = new float[rows,columns];
string test = "";
for(int r = 0; r < rows; r++){
for(int c = 0; c < columns; c++){
target[r,c] = Random.Range(0.0f, 1.0f);
test += target[r,c] + ", ";
}
}
hidden_weights.Add(target);
}
}
float activation(float x){
// tanh(x);
return (1 - Mathf.Exp (-2 * x)) / (1 + Mathf.Exp (-2 * x));
}
float[] _dot_matrix(float[] results, float[,] weights){
float[] new_matrix = new float[weights.GetLength(1)];
string t0 = "";
for (int r = 0; r < weights.GetLength(1); r++){
float res = 0;
for (int c = 0; c < weights.GetLength(0); c++) {
res += results[c] * weights[c,r];
}
new_matrix[r] = res;
}
return new_matrix;
}
float[] _map_activation(float[] pre_results){
float[] results = new float[pre_results.Length];
for (int i = 0; i < results.Length; i++) {
results[i] = activation(pre_results[i]);
}
return results;
}
float[] feed_forward(){
int h;
for (h = 0; h < n_hidden_layers + 2; h++) {
float[] dot_matrix_result;
if(h == 0){
dot_matrix_result = _dot_matrix(inputs, (float[,])hidden_weights[h]);
}
else if (h == n_hidden_layers +1){
dot_matrix_result = _dot_matrix((float[])hidden_results[h-1], (float[,])hidden_weights[h]);
output_results = _map_activation(dot_matrix_result);
break;
}
else {
dot_matrix_result = _dot_matrix((float[])hidden_results[h-1], (float[,])hidden_weights[h]);
}
float[] result = _map_activation(dot_matrix_result);
hidden_results[h] = _map_activation(result);
}
return output_results;
}
float[] normalize_input(float[] inputs){
float sum = 0.0f;
for (int i = 0; i < inputs.Length; i++) {
sum += inputs[i] ;
}
float average = sum / inputs.Length;
float[] deviations = new float[inputs.Length];
for (int i = 0; i < inputs.Length; i++) {
deviations[i] = Mathf.Pow(inputs[i] - average,2);
}
float sum_deviation = 0;
for (int i = 0; i < deviations.Length; i++) {
sum_deviation += deviations[i];
}
float variance = sum_deviation / deviations.Length;
float std = Mathf.Sqrt (variance);
for (int i = 0; i < inputs.Length; i++) {
inputs[i] = (inputs[i] - average)/std;
}
return inputs;
}
public void start_net(float[] inputs){
this.inputs = normalize_input(inputs);
feed_forward ();
}
}
I run the net from other script using the init method and then the start_net() method.
I made a test with not random weights and fixed input data, but it didn't came to the same result as the python code.
What's wrong with the C# code?
Related
You may think I am crazy, but I decided to write a neural network from scratch in C# for studying purposes. Please, be patient, I still have little experience)). English is not my first language, so I am sorry for it in advance.
I started with a program for handwritten digit recognizing with the MNIST database. I've read through a book about the algorithms inside the process and wrote this code.
public class NeuralNetwork
{
private List<Matrix<double>> weights = new List<Matrix<double>>();
private List<Vector<double>> biases = new List<Vector<double>>();
private Random random = new Random();
private List<Image> test_data;
private int[] layer_sizes;
public NeuralNetwork(params int[] layers)
{
layer_sizes = layers;
for (int i = 0; i < layers.Length - 1; i++)
{
var weigthLayer = Matrix<double>.Build.Dense(layers[i + 1], layers[i], (k, j) => random.NextDouble());
weights.Add(weigthLayer);
}
for (int i = 1; i < layers.Length; i++)
{
var biasesLayer = Vector<double>.Build.Dense(layers[i], (k) => random.NextDouble());
biases.Add(biasesLayer);
}
}
public Vector<double> FeedForward(Vector<double> a)
{
for (int i = 0; i < weights.Count; i++)
{
a = Sigmoid(weights[i].Multiply(a) + biases[i]);
}
return Sigmoid(a);
}
public void SGD(ITrainingDataProvider dataProvider, int epochs, int chuck_size, double eta)
{
test_data = new MnistReader().ReadTestData();
Console.WriteLine("SGD algorithm started");
var training_data = dataProvider.ReadTrainingData();
Console.WriteLine("Training data has beeen read");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
for (int epoch = 0; epoch < epochs; epoch++)
{
training_data = training_data.OrderBy(item => random.Next()).ToList();
List<List<Image>> chunks = training_data.ChunkBy(chuck_size);
foreach (List<Image> chunk in chunks)
{
ProcessChunk(chunk, eta);
}
Console.WriteLine($"Epoch: {epoch + 1}/{epochs}");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
}
Console.WriteLine("Done!");
}
private double Test(List<Image> data)
{
int count = 0;
foreach (Image im in data)
{
var output = FeedForward(im.DataToVector());
int number = output.MaximumIndex();
if (number == (int)im.Label)
{
count++;
}
}
return (double)count / data.Count * 100;
}
private void ProcessChunk(List<Image> chunk, double eta)
{
Delta[] deltas = new Delta[chunk.Count];
for (int i = 0; i < chunk.Count; i++)
{
Image image = chunk[i];
var input = image.DataToVector();
var desired_output = Vector<double>.Build.Dense(layer_sizes[layer_sizes.Length - 1]);
desired_output[(int)image.Label] = 1;
Delta delta = BackPropagation(input, desired_output);
deltas[i] = delta;
}
Delta sum = deltas[0];
for (int i = 1; i < deltas.Length; i++)
{
sum += deltas[i];
}
Delta average_delta = sum / deltas.Length;
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
weights[i] += average_delta.d_weights[i].Multiply(eta);
biases[i] += average_delta.d_biases[i].Multiply(eta);
}
}
private Delta BackPropagation(Vector<double> input, Vector<double> desired_output)
{
List<Vector<double>> activations = new List<Vector<double>>();
List<Vector<double>> zs = new List<Vector<double>>();
Vector<double> a = input;
activations.Add(input);
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
var z = weights[i].Multiply(a) + biases[i];
zs.Add(z);
a = Sigmoid(z);
activations.Add(a);
}
List<Vector<double>> errors = new List<Vector<double>>();
List<Matrix<double>> delta_weights = new List<Matrix<double>>();
List<Vector<double>> delta_biases = new List<Vector<double>>();
var error = CDerivative(activations[activations.Count - 1], desired_output).HProd(SigmoidDerivative(zs[^1]));
errors.Add(error);
int steps = 0;
for (int i = layer_sizes.Length - 2; i >= 1; i--)
{
var layer_error = weights[i].Transpose().Multiply(errors[steps]).HProd(SigmoidDerivative(zs[i - 1]));
errors.Add(layer_error);
steps++;
}
errors.Reverse();
for (int i = layer_sizes.Length - 1; i >= 1; i--)
{
var delta_layer_weights = (errors[i - 1].ToColumnMatrix() * activations[i - 1].ToRowMatrix()).Multiply(-1);
delta_weights.Add(delta_layer_weights);
var delta_layer_biases = errors[i - 1].Multiply(-1);
delta_biases.Add(delta_layer_biases);
}
delta_biases.Reverse();
delta_weights.Reverse();
return new Delta { d_weights = delta_weights, d_biases = delta_biases };
}
private Vector<double> CDerivative(Vector<double> x, Vector<double> y)
{
return x - y;
}
private Vector<double> Sigmoid(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = 1.0 / (1.0 + Math.Exp(-x[i]));
}
return x;
}
private Vector<double> SigmoidDerivative(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = Math.Exp(-x[i]) / Math.Pow(1.0 + Math.Exp(-x[i]), 2);
}
return x;
}
}
Delta class. A simple DTO to store weights and biases changes in a single object.
public class Delta
{
public List<Matrix<double>> d_weights { get; set; }
public List<Vector<double>> d_biases { get; set; }
public static Delta operator +(Delta d1, Delta d2)
{
Delta result = d1;
for (int i = 0; i < d2.d_weights.Count; i++)
{
result.d_weights[i] += d2.d_weights[i];
}
for (int i = 0; i < d2.d_biases.Count; i++)
{
result.d_biases[i] += d2.d_biases[i];
}
return result;
}
public static Delta operator /(Delta d1, double d)
{
Delta result = d1;
for (int i = 0; i < d1.d_weights.Count; i++)
{
result.d_weights[i] /= d;
}
for (int i = 0; i < d1.d_biases.Count; i++)
{
result.d_biases[i] /= d;
}
return result;
}
}
Everything ended up working fine, however complex networks with 1 or more hidden layers don't show any significant results. They are getting best 70% accuracy and then the learning curve drops. The accuracy returns to its 20-30%. Typically, the graph looks like a square root function, but in my case it is more like a turned around quadratic parabola the graph of my tries with different amounts of neurons in the first hidden layer
After a few tries, I found out, that without any hidden layers the algorithm works just fine. It learns up to 90% of accuracy and then the graph never falls down. Apparently, the bug is somewhere in back-propagation algorithm. It doesn't cause any problems with only input and output layers, but it does, when I add a hidden layer.
I have been trying to find the problem for a long time and I hope that someone, smarter than me, will be able to help.
Thanks in advance!
I am trying to perform Cross Correlation on these 2 images in C#:
Image, Template
Matlab says the result is supposed to look like: Matlab result
But this is my result: My result
Here is my Cross Correlation function:
public static Signal2D CrossCorrelation2D(Signal2D signal, Signal2D pulse) {
return InverseFFT2D(FFT2D(signal) * FFT2D(pulse).GetConjugate());
}
Here is my FFT2D:
public static Signal2D FFT2D(Signal2D signal) {
Signal2D result = new Signal2D(signal.Height, signal.Width);
for (int i = 0; i < result.Height; i++)
result[i] = new ComplexNumber[signal[i].Length];
//rows
for (int n = 0; n < signal.Height; n++) {
result[n] = FFT(signal[n]);
}
//columns
for (int i = 0; i < signal[0].Length; i++) {
ComplexNumber[] col = new ComplexNumber[signal.Height];
for (int j = 0; j < col.Length; j++) {
col[j] = result[j][i];
}
col = FFT(col);
for (int j = 0; j < col.Length; j++) {
result[j][i] = col[j];
}
}
return result;
}
Here is my FFT:
public static Signal FFT(Signal signal) {
int N = signal.Length;
if (N == 1)
return signal;
if ((N & (N - 1)) != 0)
throw new ArgumentOutOfRangeException("signal length must be a power of 2");
Signal evenArr = new Signal(N / 2);
Signal oddArr = new Signal(N / 2);
for (int i = 0; i < N / 2; i++) {
evenArr[i] = signal[2 * i];
}
evenArr = FFT(evenArr);
for (int i = 0; i < N / 2; i++) {
oddArr[i] = signal[2 * i + 1];
}
oddArr = FFT(oddArr);
Signal result = new Signal(N);
for (int k = 0; k < N / 2; k++) {
double w = -2.0 * k * Math.PI / N;
ComplexNumber wk = new ComplexNumber(Math.Cos(w), Math.Sin(w));
ComplexNumber even = evenArr[k];
ComplexNumber odd = oddArr[k];
result[k] = even + (wk * odd);
result[k + N / 2] = even - (wk * odd);
}
return result;
}
Here is my Signal multiplication (using pointwise multiplication):
public static Signal2D operator* (Signal2D a, Signal2D b) {
if (a.Height != b.Height || a.Width != b.Width)
throw new ArgumentException("Sizes must be equal");
Signal2D result = new Signal2D(a.Height, a.Width);
for (int y = 0; y < a.Height; y++) {
for (int x = 0; x < a.Width; x++) {
result[y][x] = a[y][x] * b[y][x];
}
}
return result;
}
Any help is appreciated, thanks.
Edit: I left the matlab image at the original size of 1023 by 1023 and overlayed my result. It looks like I may already be at the result, I am just not sure how Matlab pads the image. Overlayed results (The red is the white part from my result, the grey is the black part from my result. Black/white is from Matlab)
I'm trying to implement a parallel Gauss-Seidel algorithm. Here is my kernel function
#define dx (float2)(1,0)
#define dy (float2)(0,1)
__kernel void rbgs(read_only image2d_t fh, read_only image2d_t vh,
global float *vvh, __global float* hx, __global float* hy,__global int* red)
{
float2 id = (float2)(get_global_id(0) << 1,get_global_id(1));
const float2 sz = (float2)(1,get_global_size(0));
const sampler_t sampler = CLK_ADDRESS_CLAMP_TO_EDGE;
id.x += ((int)(id.y + *red) & 1);
const float s = ((read_imagef(vh,sampler,id-dx).x + read_imagef(vh,sampler,id+dx).x)/(*hy* *hy) + (read_imagef(vh,sampler,id-dy).x +
read_imagef(vh,sampler,id+dy).x)/(*hx* *hx) - read_imagef(fh, sampler,id).x)/(2/(*hx**hx)+2/(*hy**hy));
vvh[(int)dot(id,sz)] = s;
}
It does compile good, but I have problems with passing an Image to kernel
Here's what I try
var rows = u.GetLength(0);
var cols = u.GetLength(1);
var array1d = new float[rows * cols];
var current = 0;
for (int i = 0; i < rows; i++)
{
for (int j = 0; j < cols; j++)
{
array1d[current++] = u[i, j];
}
}
rows = fh.GetLength(0);
cols = fh.GetLength(1);
var array1df= new float[rows * cols];
current = 0;
for (int i = 0; i < rows; i++)
{
for (int j = 0; j < cols; j++)
{
array1df[current++] = fh[i, j];
}
}
CLCalc.Program.Image2D CLfh = new CLCalc.Program.Image2D(array1df, M, N);
CLCalc.Program.Image2D CLvh = new CLCalc.Program.Image2D(array1d, M, N);
CLfh.WriteToDevice(array1df);
CLvh.WriteToDevice(array1d);
float[] solution = new float[(N+1)*(M+1)];
CLCalc.Program.Variable Stepx = new CLCalc.Program.Variable(new float[] { hx });
CLCalc.Program.Variable Stepy = new CLCalc.Program.Variable(new float[] { hy });
CLCalc.Program.Variable Red = new CLCalc.Program.Variable(new float[] { 1.0f });
CLCalc.Program.Variable Result = new CLCalc.Program.Variable(solution);
CLCalc.Program.MemoryObject[] args = new CLCalc.Program.MemoryObject[] {CLfh, CLvh, Result, Stepx, Stepy, Red} ;
gs.Execute(args, new int[] { 2, 4 });
Result.ReadFromDeviceTo(solution);
CLvh.WriteToDevice(solution);
Red.WriteToDevice(new float[] { 0.0f });
gs.Execute(args, new int[] { 2, 4 });
Result.ReadFromDeviceTo(solution);
for (int m = 0; m < (M + 1) * (N + 1); m++)
u[m / (N+1), m % (N+1)] = solution[m];
return u;
The program fails with Runtime Exception: Vector Length should be 4*width*height. I know that exception was supposed to be thrown because an Image stores data in RGBA format but I do not really understand how I should overcome this problem.
Any help would be appreciated.
at the moment im trying to implement a FIR lowpass filter on a wave file. The FIR coefficients where obtained using MATLAB using a 40 order. Now i need to implement the FIR algorithm in C# and im finding it difficult to implement it.
Any help?
Thanks
How about this:
private static double[] FIR(double[] b, double[] x)
{
int M = b.Length;
int n = x.Length;
//y[n]=b0x[n]+b1x[n-1]+....bmx[n-M]
var y = new double[n];
for (int yi = 0; yi < n; yi++)
{
double t = 0.0;
for (int bi = M-1; bi >=0; bi--)
{
if (yi - bi < 0) continue;
t += b[bi] * x[yi - bi];
}
y[yi] = t;
}
return y;
}
Try this. Does it help?
static void Main()
{
var bb = new List<double> { 1, 2, 3, 4 };
var xx = new List<double> { 3, 3, 4, 5 };
var yy = func_FIR(bb, xx);
for (int i = 0; i < yy.Count; i++)
{
Console.WriteLine("y[{0}] = {1}",i,yy[i]);
}
}
public static List<double> func_FIR(List<double> b, List<double> x)
{
//y[n]=b0x[n]+b1x[n-1]+....bmx[n-M]
var y = new List<double>();
int M = b.Count;
int n = x.Count;
double t = 0.0;
for (int j = 0; j < n; j++)
{
for (int i = 0; i < M; i++)
{
t += b[i] * x[n - i-1];
}
y.Add(t);
}
return y;
}
I've done a fft to get fundamental frequency in real time and to implement high and low pass filters.
Now I want to be able to record to a .wav file after I apply a filter.
First I'll have to invert the fft and that is my question.
What are the steps to do this?
I use the FFT defined in this project.
Here is the code for it:
using System;
using System.Collections.Generic;
using System.Text;
namespace SoundLog
{
public class FourierTransform
{
static private int n, nu;
static private int BitReverse(int j)
{
int j2;
int j1 = j;
int k = 0;
for (int i = 1; i <= nu; i++)
{
j2 = j1 / 2;
k = 2 * k + j1 - 2 * j2;
j1 = j2;
}
return k;
}
static public double[] FFT(ref double[] x)
{
// Assume n is a power of 2
n = x.Length;
nu = (int)(Math.Log(n) / Math.Log(2));
int n2 = n / 2;
int nu1 = nu - 1;
double[] xre = new double[n];
double[] xim = new double[n];
double[] magnitude = new double[n2];
double[] decibel = new double[n2];
double tr, ti, p, arg, c, s;
for (int i = 0; i < n; i++)
{
xre[i] = x[i];
xim[i] = 0.0f;
}
int k = 0;
for (int l = 1; l <= nu; l++)
{
while (k < n)
{
for (int i = 1; i <= n2; i++)
{
p = BitReverse(k >> nu1);
arg = 2 * (double)Math.PI * p / n;
c = (double)Math.Cos(arg);
s = (double)Math.Sin(arg);
tr = xre[k + n2] * c + xim[k + n2] * s;
ti = xim[k + n2] * c - xre[k + n2] * s;
xre[k + n2] = xre[k] - tr;
xim[k + n2] = xim[k] - ti;
xre[k] += tr;
xim[k] += ti;
k++;
}
k += n2;
}
k = 0;
nu1--;
n2 = n2 / 2;
}
k = 0;
int r;
while (k < n)
{
r = BitReverse(k);
if (r > k)
{
tr = xre[k];
ti = xim[k];
xre[k] = xre[r];
xim[k] = xim[r];
xre[r] = tr;
xim[r] = ti;
}
k++;
}
for (int i = 0; i < n / 2; i++)
//magnitude[i] = (float)(Math.Sqrt((xre[i] * xre[i]) + (xim[i] * xim[i])));
decibel[i] = 10.0 * Math.Log10((float)(Math.Sqrt((xre[i] * xre[i]) + (xim[i] * xim[i]))));
//return magnitude;
return decibel;
}
}
}
There are so many really good fft implementations around such as FFTW that I highly recommend using one. They come with ifft as well. Yours, as implemented, will be excruciatingly slow.