You may think I am crazy, but I decided to write a neural network from scratch in C# for studying purposes. Please, be patient, I still have little experience)). English is not my first language, so I am sorry for it in advance.
I started with a program for handwritten digit recognizing with the MNIST database. I've read through a book about the algorithms inside the process and wrote this code.
public class NeuralNetwork
{
private List<Matrix<double>> weights = new List<Matrix<double>>();
private List<Vector<double>> biases = new List<Vector<double>>();
private Random random = new Random();
private List<Image> test_data;
private int[] layer_sizes;
public NeuralNetwork(params int[] layers)
{
layer_sizes = layers;
for (int i = 0; i < layers.Length - 1; i++)
{
var weigthLayer = Matrix<double>.Build.Dense(layers[i + 1], layers[i], (k, j) => random.NextDouble());
weights.Add(weigthLayer);
}
for (int i = 1; i < layers.Length; i++)
{
var biasesLayer = Vector<double>.Build.Dense(layers[i], (k) => random.NextDouble());
biases.Add(biasesLayer);
}
}
public Vector<double> FeedForward(Vector<double> a)
{
for (int i = 0; i < weights.Count; i++)
{
a = Sigmoid(weights[i].Multiply(a) + biases[i]);
}
return Sigmoid(a);
}
public void SGD(ITrainingDataProvider dataProvider, int epochs, int chuck_size, double eta)
{
test_data = new MnistReader().ReadTestData();
Console.WriteLine("SGD algorithm started");
var training_data = dataProvider.ReadTrainingData();
Console.WriteLine("Training data has beeen read");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
for (int epoch = 0; epoch < epochs; epoch++)
{
training_data = training_data.OrderBy(item => random.Next()).ToList();
List<List<Image>> chunks = training_data.ChunkBy(chuck_size);
foreach (List<Image> chunk in chunks)
{
ProcessChunk(chunk, eta);
}
Console.WriteLine($"Epoch: {epoch + 1}/{epochs}");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
}
Console.WriteLine("Done!");
}
private double Test(List<Image> data)
{
int count = 0;
foreach (Image im in data)
{
var output = FeedForward(im.DataToVector());
int number = output.MaximumIndex();
if (number == (int)im.Label)
{
count++;
}
}
return (double)count / data.Count * 100;
}
private void ProcessChunk(List<Image> chunk, double eta)
{
Delta[] deltas = new Delta[chunk.Count];
for (int i = 0; i < chunk.Count; i++)
{
Image image = chunk[i];
var input = image.DataToVector();
var desired_output = Vector<double>.Build.Dense(layer_sizes[layer_sizes.Length - 1]);
desired_output[(int)image.Label] = 1;
Delta delta = BackPropagation(input, desired_output);
deltas[i] = delta;
}
Delta sum = deltas[0];
for (int i = 1; i < deltas.Length; i++)
{
sum += deltas[i];
}
Delta average_delta = sum / deltas.Length;
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
weights[i] += average_delta.d_weights[i].Multiply(eta);
biases[i] += average_delta.d_biases[i].Multiply(eta);
}
}
private Delta BackPropagation(Vector<double> input, Vector<double> desired_output)
{
List<Vector<double>> activations = new List<Vector<double>>();
List<Vector<double>> zs = new List<Vector<double>>();
Vector<double> a = input;
activations.Add(input);
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
var z = weights[i].Multiply(a) + biases[i];
zs.Add(z);
a = Sigmoid(z);
activations.Add(a);
}
List<Vector<double>> errors = new List<Vector<double>>();
List<Matrix<double>> delta_weights = new List<Matrix<double>>();
List<Vector<double>> delta_biases = new List<Vector<double>>();
var error = CDerivative(activations[activations.Count - 1], desired_output).HProd(SigmoidDerivative(zs[^1]));
errors.Add(error);
int steps = 0;
for (int i = layer_sizes.Length - 2; i >= 1; i--)
{
var layer_error = weights[i].Transpose().Multiply(errors[steps]).HProd(SigmoidDerivative(zs[i - 1]));
errors.Add(layer_error);
steps++;
}
errors.Reverse();
for (int i = layer_sizes.Length - 1; i >= 1; i--)
{
var delta_layer_weights = (errors[i - 1].ToColumnMatrix() * activations[i - 1].ToRowMatrix()).Multiply(-1);
delta_weights.Add(delta_layer_weights);
var delta_layer_biases = errors[i - 1].Multiply(-1);
delta_biases.Add(delta_layer_biases);
}
delta_biases.Reverse();
delta_weights.Reverse();
return new Delta { d_weights = delta_weights, d_biases = delta_biases };
}
private Vector<double> CDerivative(Vector<double> x, Vector<double> y)
{
return x - y;
}
private Vector<double> Sigmoid(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = 1.0 / (1.0 + Math.Exp(-x[i]));
}
return x;
}
private Vector<double> SigmoidDerivative(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = Math.Exp(-x[i]) / Math.Pow(1.0 + Math.Exp(-x[i]), 2);
}
return x;
}
}
Delta class. A simple DTO to store weights and biases changes in a single object.
public class Delta
{
public List<Matrix<double>> d_weights { get; set; }
public List<Vector<double>> d_biases { get; set; }
public static Delta operator +(Delta d1, Delta d2)
{
Delta result = d1;
for (int i = 0; i < d2.d_weights.Count; i++)
{
result.d_weights[i] += d2.d_weights[i];
}
for (int i = 0; i < d2.d_biases.Count; i++)
{
result.d_biases[i] += d2.d_biases[i];
}
return result;
}
public static Delta operator /(Delta d1, double d)
{
Delta result = d1;
for (int i = 0; i < d1.d_weights.Count; i++)
{
result.d_weights[i] /= d;
}
for (int i = 0; i < d1.d_biases.Count; i++)
{
result.d_biases[i] /= d;
}
return result;
}
}
Everything ended up working fine, however complex networks with 1 or more hidden layers don't show any significant results. They are getting best 70% accuracy and then the learning curve drops. The accuracy returns to its 20-30%. Typically, the graph looks like a square root function, but in my case it is more like a turned around quadratic parabola the graph of my tries with different amounts of neurons in the first hidden layer
After a few tries, I found out, that without any hidden layers the algorithm works just fine. It learns up to 90% of accuracy and then the graph never falls down. Apparently, the bug is somewhere in back-propagation algorithm. It doesn't cause any problems with only input and output layers, but it does, when I add a hidden layer.
I have been trying to find the problem for a long time and I hope that someone, smarter than me, will be able to help.
Thanks in advance!
Related
I am working on a project that compares the time bubble and selection sort take. I made two separate programs and combined them into one and now bubble sort is running much faster than selection sort. I checked to make sure that the code wasn't just giving me 0s because of some conversion error and was running as intended. I am using System.Diagnostics; to measure the time. I also checked that the machine was not the problem, I ran it on Replit and got similar results.
{
class Program
{
public static int s1 = 0;
public static int s2 = 0;
static decimal bubblesort(int[] arr1)
{
int n = arr1.Length;
var sw1 = Stopwatch.StartNew();
for (int i = 0; i < n - 1; i++)
{
for (int j = 0; j < n - i - 1; j++)
{
if (arr1[j] > arr1[j + 1])
{
int tmp = arr1[j];
// swap tmp and arr[i] int tmp = arr[j];
arr1[j] = arr1[j + 1];
arr1[j + 1] = tmp;
s1++;
}
}
}
sw1.Stop();
// Console.WriteLine(sw1.ElapsedMilliseconds);
decimal a = Convert.ToDecimal(sw1.ElapsedMilliseconds);
return a;
}
static decimal selectionsort(int[] arr2)
{
int n = arr2.Length;
var sw1 = Stopwatch.StartNew();
// for (int e = 0; e < 1000; e++)
// {
for (int x = 0; x < arr2.Length - 1; x++)
{
int minPos = x;
for (int y = x + 1; y < arr2.Length; y++)
{
if (arr2[y] < arr2[minPos])
minPos = y;
}
if (x != minPos && minPos < arr2.Length)
{
int temp = arr2[minPos];
arr2[minPos] = arr2[x];
arr2[x] = temp;
s2++;
}
}
// }
sw1.Stop();
// Console.WriteLine(sw1.ElapsedMilliseconds);
decimal a = Convert.ToDecimal(sw1.ElapsedMilliseconds);
return a;
}
static void Main(string[] args)
{
Console.WriteLine("Enter the size of n");
int n = Convert.ToInt32(Console.ReadLine());
Random rnd = new System.Random();
decimal bs = 0M;
decimal ss = 0M;
int s = 0;
int[] arr1 = new int[n];
int tx = 1000; //tx is a variable that I can use to adjust sample size
decimal tm = Convert.ToDecimal(tx);
for (int i = 0; i < tx; i++)
{
for (int a = 0; a < n; a++)
{
arr1[a] = rnd.Next(0, 1000000);
}
ss += selectionsort(arr1);
bs += bubblesort(arr1);
}
bs = bs / tm;
ss = ss / tm;
Console.WriteLine("Bubble Sort took " + bs + " miliseconds");
Console.WriteLine("Selection Sort took " + ss + " miliseconds");
}
}
}
What is going on? What is causing bubble sort to be fast or what is slowing down Selection sort? How can I fix this?
I found that the problem was that the Selection Sort was looping 1000 times per method run in addition to the 1000 runs for sample size, causing the method to perform significantly worse than bubble sort. Thank you guys for help and thank you TheGeneral for showing me the benchmarking tools. Also, the array that was given as a parameter was a copy instead of a reference, as running through the loop manually showed me that the bubble sort was doing it's job and not sorting an already sorted array.
To solve your initial problem you just need to copy your arrays, you can do this easily with ToArray():
Creates an array from a IEnumerable.
ss += selectionsort(arr1.ToArray());
bs += bubblesort(arr1.ToArray());
However let's learn how to do a more reliable benchmark with BenchmarkDotNet:
BenchmarkDotNet Nuget
Official Documentation
Given
public class Sort
{
public static void BubbleSort(int[] arr1)
{
int n = arr1.Length;
for (int i = 0; i < n - 1; i++)
{
for (int j = 0; j < n - i - 1; j++)
{
if (arr1[j] > arr1[j + 1])
{
int tmp = arr1[j];
// swap tmp and arr[i] int tmp = arr[j];
arr1[j] = arr1[j + 1];
arr1[j + 1] = tmp;
}
}
}
}
public static void SelectionSort(int[] arr2)
{
int n = arr2.Length;
for (int x = 0; x < arr2.Length - 1; x++)
{
int minPos = x;
for (int y = x + 1; y < arr2.Length; y++)
{
if (arr2[y] < arr2[minPos])
minPos = y;
}
if (x != minPos && minPos < arr2.Length)
{
int temp = arr2[minPos];
arr2[minPos] = arr2[x];
arr2[x] = temp;
}
}
}
}
Benchmark code
[SimpleJob(RuntimeMoniker.Net50)]
[MemoryDiagnoser()]
public class SortBenchmark
{
private int[] data;
[Params(100, 1000)]
public int N;
[GlobalSetup]
public void Setup()
{
var r = new Random(42);
data = Enumerable
.Repeat(0, N)
.Select(i => r.Next(0, N))
.ToArray();
}
[Benchmark]
public void Bubble() => Sort.BubbleSort(data.ToArray());
[Benchmark]
public void Selection() => Sort.SelectionSort(data.ToArray());
}
Usage
static void Main(string[] args)
{
BenchmarkRunner.Run<SortBenchmark>();
}
Results
Method
N
Mean
Error
StdDev
Bubble
100
8.553 us
0.0753 us
0.0704 us
Selection
100
4.757 us
0.0247 us
0.0231 us
Bubble
1000
657.760 us
7.2581 us
6.7893 us
Selection
1000
300.395 us
2.3302 us
2.1796 us
Summary
What have we learnt? Your bubble sort code is slower ¯\_(ツ)_/¯
It looks like you're passing in the sorted array into Bubble Sort. Because arrays are passed by reference, the sort that you're doing on the array is editing the same contents of the array that will be eventually passed into bubble sort.
Make a second array and pass the second array into bubble sort.
I'm a hobby programmer.
I tried to ask this question earlier on a very unstructured way (Sorry again), now I try to ask on the proper way.
I wrote the following code that seems to work unreliably.
The code was written like this for several reasons. I know it's messy but it should still work. To explain why I wrote it like this would mean that I need to explain several weeks' of work that is quite extensive. Please accept that this is at least the least worse option I could figure out. In the below sample I removed all sections of the code that are not needed to reproduce the error.
What this program does in a nutshell:
The purpose is to check a large number of parameter combinations for a program that receives streaming data. I simulate the original process to test parameter combinations.
First data is read from files that represents recorded streaming data.
Then the data is aggregated.
Then I build a list of parameters to test for.
Finally I run the code for each parameter combination in parallel.
Inside the parallel part I calculate a financial indicator called the bollinger bands. This is a moving average with adding +/- standard deviation. This means the upper line and the lower line should only be equal when variable bBandDelta = 0. However sometimes it happens that CandleList[slot, w][ctr].bollingerUp is equal to CandleList[slot, w][ctr].bollingerDown even when bBandDelta is not 0.
As a result I don't understand how can line 277 kick in. It seems that sometimes the program fails to write to the CandleList[slot, w][ctr]. However this should not be possible because (1) I lock the list and (2) I use ConcurrentBag. Could I have some help please?
Source files are here.
The code is:
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;
namespace Justfortest
{
class tick : IComparable<tick> //Data element to represent a tick
{
public string disp_name; //ticker ID
public DateTime? trd_date; //trade date
public TimeSpan? trdtim_1; //trade time
public decimal trdprc_1; //price
public int? trdvol_1; //tick volume
public int CompareTo(tick other)
{
if (this.trdprc_1 == other.trdprc_1)
{
return other.trdprc_1.CompareTo(this.trdprc_1); //Return the later item
}
return this.trdprc_1.CompareTo(other.trdprc_1); //Return the earlier item
}
}
class candle : IComparable<candle> //Data element to represent a candle and all chart data calculated on candle level
{
public int id = 0;
public DateTime? openDate;
public TimeSpan? openTime;
public DateTime? closeDate;
public TimeSpan? closeTime;
public decimal open = 0;
public decimal high = 0;
public decimal low = 0;
public decimal close = 0;
public int? volume = 0;
public decimal totalPrice = 0;
public decimal bollingerUp = 0; //Bollinger upper line
public decimal bollingerDown = 0; //Bollinger below line
public int CompareTo(candle other)
{
if (totalPrice == other.totalPrice)
{
return other.totalPrice.CompareTo(totalPrice); //Return the later item
}
return totalPrice.CompareTo(other.totalPrice); //Return the earlier item
}
}
class param : IComparable<param> //Data element represent a trade event signal
{
public int par1;
public int bollPar;
public int par2;
public int par3;
public int par4;
public int par5;
public int par6;
public decimal par7;
public decimal par8;
public decimal par9;
public decimal par10;
int IComparable<param>.CompareTo(param other)
{
throw new NotImplementedException();
}
}
class programCLass
{
void myProgram()
{
Console.WriteLine("Hello");
Console.WindowWidth = 180;
string[] sources = new string[]
{
#"C:\test\source\sourceW1.csv",
#"C:\test\source\sourceW2.csv",
};
List<candle>[] sourceCandleList = new List<candle>[sources.Count()];
List<param> paramList = new List<param>(10000000);
var csvAnalyzer = new StringBuilder();
{
List<tick>[] updatelist = new List<tick>[sources.Count()];
Console.WriteLine("START LOAD");
for (var i = 0; i < sources.Count(); i++)
{
var file = sources[i];
updatelist[i] = new List<tick>();
// ---------- Read CSV file ----------
var reader = new StreamReader(File.OpenRead(file));
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(',');
tick update = new tick();
update.disp_name = values[0].ToString();
update.trd_date = Convert.ToDateTime(values[1]);
update.trdtim_1 = TimeSpan.Parse(values[2]);
update.trdprc_1 = Convert.ToDecimal(values[3]);
update.trdvol_1 = Convert.ToInt32(values[4]);
updatelist[i].Add(update);
}
Console.WriteLine(i);
}
Console.WriteLine("END LOAD"); // All files are in the memory
// Aggreagate
Console.WriteLine("AGGREGATE START");
int tickAggr = 500;
for (var w = 0; w < sources.Count(); w++)
{
sourceCandleList[w] = new List<candle>();
List<tick> FuturesList = new List<tick>();
foreach (var update in updatelist[w])
{
tick t = new tick();
t.disp_name = update.disp_name.ToString();
t.trd_date = update.trd_date;
t.trdtim_1 = update.trdtim_1;
t.trdprc_1 = Convert.ToDecimal(update.trdprc_1);
t.trdvol_1 = update.trdvol_1;
// Add new tick to the list
FuturesList.Add(t);
if (FuturesList.Count == Math.Truncate(FuturesList.Count / (decimal)tickAggr) * tickAggr)
{
candle c = new candle();
c.openDate = FuturesList[FuturesList.Count - tickAggr].trd_date;
c.openTime = FuturesList[FuturesList.Count - tickAggr].trdtim_1;
c.closeDate = FuturesList.Last().trd_date;
c.closeTime = FuturesList.Last().trdtim_1;
c.open = FuturesList[FuturesList.Count - tickAggr].trdprc_1;
c.high = FuturesList.GetRange(FuturesList.Count - tickAggr, tickAggr).Max().trdprc_1;
c.low = FuturesList.GetRange(FuturesList.Count - tickAggr, tickAggr).Min().trdprc_1;
c.close = FuturesList.Last().trdprc_1;
c.volume = FuturesList.GetRange(FuturesList.Count - tickAggr, tickAggr).Sum(tick => tick.trdvol_1);
c.totalPrice = (c.open + c.high + c.low + c.close) / 4;
sourceCandleList[w].Add(c);
if (sourceCandleList[w].Count == 1)
{
c.id = 0;
}
else
{
c.id = sourceCandleList[w][sourceCandleList[w].Count - 2].id + 1;
}
}
}
FuturesList.Clear();
}
Console.WriteLine("AGGREGATE END");
for (var i = 0; i < sources.Count(); i++)
{
updatelist[i].Clear();
}
}
Console.WriteLine("BUILD PARAMLIST");
for (int par1 = 8; par1 <= 20; par1 += 4) // parameter deployed
{
for (int bollPar = 10; bollPar <= 25; bollPar += 5) // parameter deployed
{
for (int par2 = 6; par2 <= 18; par2 += 4) // parameter deployed
{
for (int par3 = 14; par3 <= 20; par3 += 3) // parameter deployed
{
for (int par4 = 10; par4 <= 20; par4 += 5) // parameter deployed
{
for (int par5 = 4; par5 <= 10; par5 += 2) // parameter deployed
{
for (int par6 = 5; par6 <= 30; par6 += 5)
{
for (decimal par7 = 1.0005M; par7 <= 1.002M; par7 += 0.0005M)
{
for (decimal par8 = 1.002M; par8 <= 1.0048M; par8 += 0.0007M)
{
for (decimal par9 = 0.2M; par9 <= 0.5M; par9 += 0.1M)
{
for (decimal par10 = 0.5M; par10 <= 2; par10 += 0.5M)
{
param p = new param();
p.par1 = par1;
p.bollPar = bollPar;
p.par2 = par2;
p.par3 = par3;
p.par4 = par4;
p.par5 = par5;
p.par6 = par6;
p.par7 = par7;
p.par8 = par8;
p.par9 = par9;
p.par10 = par10;
paramList.Add(p);
}
}
}
}
}
}
}
}
}
}
}
Console.WriteLine("END BUILD PARAMLIST, scenarios to test:{0}", paramList.Count);
var sourceCount = sources.Count();
sources = null;
Console.WriteLine("Start building pools");
int maxThreads = 64;
ConcurrentBag<int> pool = new ConcurrentBag<int>();
List<candle>[,] CandleList = new List<candle>[maxThreads, sourceCount];
for (int i = 0; i <= maxThreads - 1; i++)
{
pool.Add(i);
for (int w = 0; w <= sourceCount - 1; w++)
{
CandleList[i, w] = sourceCandleList[w].ConvertAll(p => p);
}
}
Console.WriteLine("End building pools");
int pItemsProcessed = 0;
Parallel.ForEach(paramList,
new ParallelOptions { MaxDegreeOfParallelism = maxThreads },
p =>
{
int slot = 1000;
while (!pool.TryTake(out slot));
var bollPar = p.bollPar;
decimal bollingerMiddle = 0;
double bBandDeltaX = 0;
for (var w = 0; w < sourceCount; w++)
{
lock (CandleList[slot, w])
{
for (var ctr = 0; ctr < CandleList[slot, w].Count; ctr++)
{
CandleList[slot, w][ctr].bollingerUp = 0; //Bollinger upper line
CandleList[slot, w][ctr].bollingerDown = 0; //Bollinger below line
//Bollinger Bands Calculation
if (ctr + 1 >= bollPar)
{
bollingerMiddle = 0;
bBandDeltaX = 0;
for (int i = 0; i <= bollPar - 1; i++)
{
bollingerMiddle = bollingerMiddle + CandleList[slot, w][ctr - i].totalPrice;
}
bollingerMiddle = bollingerMiddle / bollPar; //average
for (int i = 0; i <= bollPar - 1; i++)
{
bBandDeltaX = bBandDeltaX + (double)Math.Pow(System.Convert.ToDouble(CandleList[slot, w][ctr - i].totalPrice) - System.Convert.ToDouble(bollingerMiddle), 2);
}
bBandDeltaX = bBandDeltaX / bollPar;
decimal bBandDelta = (decimal)Math.Sqrt(System.Convert.ToDouble(bBandDeltaX));
CandleList[slot, w][ctr].bollingerUp = bollingerMiddle + 2 * bBandDelta;
CandleList[slot, w][ctr].bollingerDown = bollingerMiddle - 2 * bBandDelta;
if (CandleList[slot, w][ctr].bollingerUp == CandleList[slot, w][ctr].bollingerDown)
{
Console.WriteLine("?! Items processed=" + pItemsProcessed + " bollPar=" + bollPar + " ctr=" + ctr + " bollingerMiddle=" + bollingerMiddle + " bBandDeltaX=" + bBandDeltaX + " bBandDelta=" + bBandDelta + " bollingerUp=" + CandleList[slot, w][ctr].bollingerUp + " bollingerDown=" + CandleList[slot, w][ctr].bollingerDown);
}
}
// REMOVED Further calculations happen here
}
// REMOVED Some evaluations happen here
}
}
// REMOVED Some more evaluations happen here
Interlocked.Increment(ref pItemsProcessed);
pool.Add(slot);
});
}
static void Main(string[] args)
{
var P = new programCLass();
P.myProgram();
}
}
}
I'm attempting to make a Neural Network in C#, I based the design in a python code I made a while back. But somehow the end result is not the same.
I'm new to C# and I'm using it in Unity, so I have limitation to library uses.
In python numpy can do matrix multiplications with the numpy.dot() method. I Haven't found something similar in C#, especially in Unity. So I had to do it by hand.
The Python code:
import numpy as np
class NN:
def __init__(self, n_input, n_hidden_layers, n_hidden_nodes, n_output):
self.weights_hidden = []
for n in range(n_hidden_layers + 1):
if n == 0:
size = n_input, n_hidden_nodes
elif n == n_hidden_layers:
size = n_hidden_nodes, n_output
else:
size = n_hidden_nodes, n_hidden_nodes
self.weights_hidden.append(
np.random.random(size)
)
#staticmethod
def activation(x):
return np.tanh(x)
def feed_forward(self, ip):
input_values = (ip - np.mean(ip, axis=0)) / np.std(ip, axis=0)
for w, weights in enumerate(self.weights_hidden):
if w == 0:
result = input_values
result = np.array(
map(self.activation, result.dot(weights))
)
return result
ANN = NN(n_input=5, n_hidden_layers=2, n_hidden_nodes=3, n_output=1)
print ANN.feed_forward([1, 2, 3, 4, 5])
My attempt to convert it to C#.
using UnityEngine;
using System.Collections;
public class neural_net : MonoBehaviour {
int n_inputs;
int n_hidden_layers;
int n_hidden_nodes;
int n_outputs;
float[] inputs;
ArrayList hidden_weights;
ArrayList hidden_results;
float[] output_results;
public void init(int n_inputs, int n_hidden_layers, int n_hidden_nodes, int n_outputs){
this.n_inputs = n_inputs;
this.n_hidden_layers = n_hidden_layers;
this.n_hidden_nodes = n_hidden_nodes;
this.n_outputs = n_outputs;
this.hidden_weights = new ArrayList ();
this.hidden_results = new ArrayList ();
this.output_results = new float[n_outputs];
int rows;
int columns;
for (int h = 0; h < n_hidden_layers + 2; h++) {
if (h == 0){
// input -> hidden
rows = n_inputs;
columns = n_hidden_nodes;
}
else if(h == n_hidden_layers + 1){
// hidden -> output
rows = n_hidden_nodes;
columns = n_outputs;
}
else {
// hidden -> hidden
rows = n_hidden_nodes;
columns = n_hidden_nodes;
}
float[] hidden_result = new float[rows*columns];
hidden_results.Add(hidden_results);
float[,] target = new float[rows,columns];
string test = "";
for(int r = 0; r < rows; r++){
for(int c = 0; c < columns; c++){
target[r,c] = Random.Range(0.0f, 1.0f);
test += target[r,c] + ", ";
}
}
hidden_weights.Add(target);
}
}
float activation(float x){
// tanh(x);
return (1 - Mathf.Exp (-2 * x)) / (1 + Mathf.Exp (-2 * x));
}
float[] _dot_matrix(float[] results, float[,] weights){
float[] new_matrix = new float[weights.GetLength(1)];
string t0 = "";
for (int r = 0; r < weights.GetLength(1); r++){
float res = 0;
for (int c = 0; c < weights.GetLength(0); c++) {
res += results[c] * weights[c,r];
}
new_matrix[r] = res;
}
return new_matrix;
}
float[] _map_activation(float[] pre_results){
float[] results = new float[pre_results.Length];
for (int i = 0; i < results.Length; i++) {
results[i] = activation(pre_results[i]);
}
return results;
}
float[] feed_forward(){
int h;
for (h = 0; h < n_hidden_layers + 2; h++) {
float[] dot_matrix_result;
if(h == 0){
dot_matrix_result = _dot_matrix(inputs, (float[,])hidden_weights[h]);
}
else if (h == n_hidden_layers +1){
dot_matrix_result = _dot_matrix((float[])hidden_results[h-1], (float[,])hidden_weights[h]);
output_results = _map_activation(dot_matrix_result);
break;
}
else {
dot_matrix_result = _dot_matrix((float[])hidden_results[h-1], (float[,])hidden_weights[h]);
}
float[] result = _map_activation(dot_matrix_result);
hidden_results[h] = _map_activation(result);
}
return output_results;
}
float[] normalize_input(float[] inputs){
float sum = 0.0f;
for (int i = 0; i < inputs.Length; i++) {
sum += inputs[i] ;
}
float average = sum / inputs.Length;
float[] deviations = new float[inputs.Length];
for (int i = 0; i < inputs.Length; i++) {
deviations[i] = Mathf.Pow(inputs[i] - average,2);
}
float sum_deviation = 0;
for (int i = 0; i < deviations.Length; i++) {
sum_deviation += deviations[i];
}
float variance = sum_deviation / deviations.Length;
float std = Mathf.Sqrt (variance);
for (int i = 0; i < inputs.Length; i++) {
inputs[i] = (inputs[i] - average)/std;
}
return inputs;
}
public void start_net(float[] inputs){
this.inputs = normalize_input(inputs);
feed_forward ();
}
}
I run the net from other script using the init method and then the start_net() method.
I made a test with not random weights and fixed input data, but it didn't came to the same result as the python code.
What's wrong with the C# code?
I am trying to create a neural network for the function y=e^(-(x-u)^2)/(2*o^2)) where u = 50 and o = 15.
I must train my neural network so I can find the 2 x's for each y. I have created the folling code, it seems to learn it nicely, but once I test the outputs go I only get numbers around 0.99 to 1 where I should get 25 and 75 and I just can't see why. My best guess is that my error correction is wrong, but can't find the error. The neural network uses back-propagation.
The test code and training set
class Program
{
static void Main(string[] args)
{
args = new string[] {
"c:\\testTrain.csv",
"c:\\testValues.csv"
};
// Output File
string fileTrainPath = null;
string fileValuesPath = null;
if (args.Length > 0)
{
fileTrainPath = args[0];
if (File.Exists(fileTrainPath))
File.Delete(fileTrainPath);
fileValuesPath = args[1];
if (File.Exists(fileValuesPath))
File.Delete(fileValuesPath);
}
double learningRate = 0.1;
double u = 50;
double o = 15;
Random rand = new Random();
Network net = new Network(1, 8, 4, 2);
NetworkTrainer netTrainer = new NetworkTrainer(learningRate, net);
List<TrainerSet> TrainerSets = new List<TrainerSet>();
for(int i = 0; i <= 20; i++)
{
double random = rand.NextDouble();
TrainerSets.Add(new TrainerSet(){
Inputs = new double[] { random },
Outputs = getX(random, u, o)
});
}
// Train Network
string fileTrainValue = String.Empty;
for (int i = 0; i <= 10000; i++)
{
if (i == 5000)
{ }
double error = netTrainer.RunEpoch(TrainerSets);
Console.WriteLine("Epoch " + i + ": Error = " + error);
if(fileTrainPath != null)
fileTrainValue += i + "," + learningRate + "," + error + "\n";
}
if (fileTrainPath != null)
File.WriteAllText(fileTrainPath, fileTrainValue);
// Test Network
string fileValuesValue = String.Empty;
for (int i = 0; i <= 100; i++)
{
double y = rand.NextDouble();
double[] dOutput = getX(y, u, o);
double[] Output = net.Compute(new double[] { y });
if (fileValuesPath != null)
fileValuesValue += i + "," + y + "," + dOutput[0] + "," + dOutput[1] + "," + Output[0] + "," + Output[1] + "\n";
}
if (fileValuesPath != null)
File.WriteAllText(fileValuesPath, fileValuesValue);
}
public static double getResult(int x, double u, double o)
{
return Math.Exp(-Math.Pow(x-u,2)/(2*Math.Pow(o,2)));
}
public static double[] getX(double y, double u, double o)
{
return new double[] {
u + Math.Sqrt(2 * Math.Pow(o, 2) * Math.Log(1/y)),
u - Math.Sqrt(2 * Math.Pow(o, 2) * Math.Log(1/y)),
};
}
}
The code behind the network
public class Network
{
protected int inputsCount;
protected int layersCount;
protected NetworkLayer[] layers;
protected double[] output;
public int Count
{
get
{
return layers.Count();
}
}
public NetworkLayer this[int index]
{
get { return layers[index]; }
}
public Network(int inputsCount, params int[] neuronsCount)
{
this.inputsCount = Math.Max(1, inputsCount);
this.layersCount = Math.Max(1, neuronsCount.Length);
layers = new NetworkLayer[neuronsCount.Length];
for (int i = 0; i < layersCount; i++)
layers[i] = new NetworkLayer(neuronsCount[i],
(i == 0) ? inputsCount : neuronsCount[i - 1]);
}
public virtual double[] Compute(double[] input)
{
output = input;
foreach (NetworkLayer layer in layers)
output = layer.Compute(output);
return output;
}
}
public class NetworkLayer
{
protected int inputsCount = 0;
protected int neuronsCount = 0;
protected Neuron[] neurons;
protected double[] output;
public Neuron this[int index]
{
get { return neurons[index]; }
}
public int Count
{
get { return neurons.Length; }
}
public int Inputs
{
get { return inputsCount; }
}
public double[] Output
{
get { return output; }
}
public NetworkLayer(int neuronsCount, int inputsCount)
{
this.inputsCount = Math.Max( 1, inputsCount );
this.neuronsCount = Math.Max( 1, neuronsCount );
neurons = new Neuron[this.neuronsCount];
output = new double[this.neuronsCount];
// create each neuron
for (int i = 0; i < neuronsCount; i++)
neurons[i] = new Neuron(inputsCount);
}
public virtual double[] Compute(double[] input)
{
// compute each neuron
for (int i = 0; i < neuronsCount; i++)
output[i] = neurons[i].Compute(input);
return output;
}
}
public class Neuron
{
protected static Random rand = new Random((int)DateTime.Now.Ticks);
public int Inputs;
public double[] Input;
public double[] Weights;
public double Output = 0;
public double Threshold;
public double Error;
public Neuron(int inputs)
{
this.Inputs = inputs;
Weights = new double[inputs];
for (int i = 0; i < inputs; i++)
Weights[i] = rand.NextDouble() * 0.5;
}
public double Compute(double[] inputs)
{
Input = inputs;
double e = 0.0;
for (int i = 0; i < inputs.Length; i++)
e += Weights[i] * inputs[i];
e -= Threshold;
return (Output = sigmoid(e));
}
private double sigmoid(double value)
{
return (1 / (1 + Math.Exp(-1 * value)));
//return 1 / (1 + Math.Exp(-value));
}
}
My Trainer
public class NetworkTrainer
{
private Network network;
private double learningRate = 0.1;
public NetworkTrainer(double a, Network network)
{
this.network = network;
this.learningRate = a;
}
public double Run(double[] input, double[] output)
{
network.Compute(input);
return CorrectErrors(output);
}
public double RunEpoch(List<TrainerSet> sets)
{
double error = 0.0;
for (int i = 0, n = sets.Count; i < n; i++)
error += Run(sets[i].Inputs, sets[i].Outputs);
// return summary error
return error;
}
private double CorrectErrors(double[] desiredOutput)
{
double[] errorLast = new double[desiredOutput.Length];
NetworkLayer lastLayer = network[network.Count - 1];
for (int i = 0; i < desiredOutput.Length; i++)
{
// S(p)=y(p)*[1-y(p)]*(yd(p)-y(p))
lastLayer[i].Error = lastLayer[i].Output * (1-lastLayer[i].Output)*(desiredOutput[i] - lastLayer[i].Output);
errorLast[i] = lastLayer[i].Error;
}
// Calculate errors
for (int l = network.Count - 2; l >= 0; l--)
{
for (int n = 0; n < network[l].Count; n++)
{
double newError = 0;
for (int np = 0; np < network[l + 1].Count; np++)
{
newError += network[l + 1][np].Weights[n] * network[l + 1][np].Error;
}
network[l][n].Error = newError;
}
}
// Update Weights
// w = w + (a * input * error)
for (int l = network.Count - 1; l >= 0; l--)
{
for (int n = 0; n < network[l].Count; n++)
{
for (int i = 0; i < network[l][n].Inputs; i++)
{
// deltaW = a * y(p) * s(p)
double deltaW = learningRate * network[l][n].Output * network[l][n].Error;
network[l][n].Weights[i] += deltaW;
}
}
}
double returnError = 0;
foreach (double e in errorLast)
returnError += e;
return returnError;
}
}
For regression problems your output layer should have the identity (or at least a linear) activation function. This way you don't have to scale your output. The derivative of the identity function is 1 and thus the derivative dE/da_i for the output layer is y-t (lastLayer[i].Output - desiredOutput[i]).
I need to calculate the factorial of numbers up to around 100! in order to determine if a series of coin flip-style data is random, as per this Wikipedia entry on Bayesian probability. As you can see there, the necessary formula involves 3 factorial calculations (but, interestingly, two of those factorial calculations are calculated along the way to the third).
I saw this question here, but I'd think that integer is going to get blown out pretty quickly. I could also make a function that is more intelligent about the factorial calculation (ie, if I have 11!/(7!3!), as per the wiki example, I could go to (11*10*9*8)/3!), but that smacks of premature optimization to me, in the sense that I want it to work, but I don't care about speed (yet).
So what's a good C# library I can call to calculate the factorial in order to get that probability? I'm not interested in all the awesomeness that can go into factorial calculation, I just want the result in a way that I can manipulate it. There does not appear to be a factorial function in the Math namespace, hence the question.
You could try Math.NET - I haven't used that library, but they do list Factorial and Logarithmic Factorial.
There has been a previous question on a similar topic. Someone there linked the Fast Factorial Functions web site, which includes some explanations of efficient algorithms and even C# source code.
Do you want to calculate factorials, or binomial coefficients?
It sounds like you want to calculate binomial coefficients - especially as you mention 11!/(7!3!).
There may be a library that can do this for you, but as a (presumably) programmer visiting stack overflow there's no reason not to write one yourself. It's not too complicated.
To avoid memory overflow, don't evaluate the result until all common factors are removed.
This algorithm still needs to be improved, but you have the basis for a good algorithm here. The denominator values need to be split into their prime factors for the best result. As it stands, this will run for n = 50 quite quickly.
float CalculateBinomial(int n, int k)
{
var numerator = new List<int>();
var denominator = new List<int>();
var denominatorOld = new List<int>();
// again ignore the k! common terms
for (int i = k + 1; i <= n; i++)
numerator.Add(i);
for (int i = 1; i <= (n - k); i++)
{
denominator.AddRange(SplitIntoPrimeFactors(i));
}
// remove all common factors
int remainder;
for (int i = 0; i < numerator.Count(); i++)
{
for (int j = 0; j < denominator.Count()
&& numerator[i] >= denominator[j]; j++)
{
if (denominator[j] > 1)
{
int result = Math.DivRem(numerator[i], denominator[j], out remainder);
if (remainder == 0)
{
numerator[i] = result;
denominator[j] = 1;
}
}
}
}
float denominatorResult = 1;
float numeratorResult = 1;
denominator.RemoveAll(x => x == 1);
numerator.RemoveAll(x => x == 1);
denominator.ForEach(d => denominatorResult = denominatorResult * d);
numerator.ForEach(num => numeratorResult = numeratorResult * num);
return numeratorResult / denominatorResult;
}
static List<int> Primes = new List<int>() { 2, 3, 5, 7, 11, 13, 17, 19,
23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97 };
List<int> SplitIntoPrimeFactors(int x)
{
var results = new List<int>();
int remainder = 0;
int i = 0;
while (!Primes.Contains(x) && x != 1)
{
int result = Math.DivRem(x, Primes[i], out remainder);
if (remainder == 0)
{
results.Add(Primes[i]);
x = result;
i = 0;
}
else
{
i++;
}
}
results.Add(x);
return results;
}
I can estimate n = 110, k = 50 (returns 6x10^31) but cannot run n = 120, k = 50.
The following can calculate the factorial of 5000 in 1 second.
public class Number
{
#region Fields
private static long _valueDivision = 1000000000;
private static int _valueDivisionDigitCount = 9;
private static string _formatZeros = "000000000";
List<long> _value;
#endregion
#region Properties
public int ValueCount { get { return _value.Count; } }
public long ValueAsLong
{
get
{
return long.Parse(ToString());
}
set { SetValue(value.ToString()); }
}
#endregion
#region Constructors
public Number()
{
_value = new List<long>();
}
public Number(long value)
: this()
{
SetValue(value.ToString());
}
public Number(string value)
: this()
{
SetValue(value);
}
private Number(List<long> list)
{
_value = list;
}
#endregion
#region Public Methods
public void SetValue(string value)
{
_value.Clear();
bool finished = false;
while (!finished)
{
if (value.Length > _valueDivisionDigitCount)
{
_value.Add(long.Parse(value.Substring(value.Length - _valueDivisionDigitCount)));
value = value.Remove(value.Length - _valueDivisionDigitCount, _valueDivisionDigitCount);
}
else
{
_value.Add(long.Parse(value));
finished = true;
}
}
}
#endregion
#region Static Methods
public static Number operator +(Number c1, Number c2)
{
return Add(c1, c2);
}
public static Number operator *(Number c1, Number c2)
{
return Mul(c1, c2);
}
private static Number Add(Number value1, Number value2)
{
Number result = new Number();
int count = Math.Max(value1._value.Count, value2._value.Count);
long reminder = 0;
long firstValue, secondValue;
for (int i = 0; i < count; i++)
{
firstValue = 0;
secondValue = 0;
if (value1._value.Count > i)
{
firstValue = value1._value[i];
}
if (value2._value.Count > i)
{
secondValue = value2._value[i];
}
reminder += firstValue + secondValue;
result._value.Add(reminder % _valueDivision);
reminder /= _valueDivision;
}
while (reminder > 0)
{
result._value.Add(reminder % _valueDivision);
reminder /= _valueDivision;
}
return result;
}
private static Number Mul(Number value1, Number value2)
{
List<List<long>> values = new List<List<long>>();
for (int i = 0; i < value2._value.Count; i++)
{
values.Add(new List<long>());
long lastremain = 0;
for (int j = 0; j < value1._value.Count; j++)
{
values[i].Add(((value1._value[j] * value2._value[i] + lastremain) % _valueDivision));
lastremain = ((value1._value[j] * value2._value[i] + lastremain) / _valueDivision);
//result.Add(();
}
while (lastremain > 0)
{
values[i].Add((lastremain % _valueDivision));
lastremain /= _valueDivision;
}
}
List<long> result = new List<long>();
for (int i = 0; i < values.Count; i++)
{
for (int j = 0; j < i; j++)
{
values[i].Insert(0, 0);
}
}
int count = values.Select(list => list.Count).Max();
int index = 0;
long lastRemain = 0;
while (count > 0)
{
for (int i = 0; i < values.Count; i++)
{
if (values[i].Count > index)
lastRemain += values[i][index];
}
result.Add((lastRemain % _valueDivision));
lastRemain /= _valueDivision;
count -= 1;
index += 1;
}
while (lastRemain > 0)
{
result.Add((lastRemain % _valueDivision));
lastRemain /= _valueDivision;
}
return new Number(result);
}
#endregion
#region Overriden Methods Of Object
public override string ToString()
{
string result = string.Empty;
for (int i = 0; i < _value.Count; i++)
{
result = _value[i].ToString(_formatZeros) + result;
}
return result.TrimStart('0');
}
#endregion
}
class Program
{
static void Main(string[] args)
{
Number number1 = new Number(5000);
DateTime dateTime = DateTime.Now;
string s = Factorial(number1).ToString();
TimeSpan timeSpan = DateTime.Now - dateTime;
long sum = s.Select(c => (long) (c - '0')).Sum();
}
static Number Factorial(Number value)
{
if( value.ValueCount==1 && value.ValueAsLong==2)
{
return value;
}
return Factorial(new Number(value.ValueAsLong - 1)) * value;
}
}
using System;
//calculating factorial with recursion
namespace ConsoleApplication2
{
class Program
{
long fun(long a)
{
if (a <= 1)
{
return 1;}
else
{
long c = a * fun(a - 1);
return c;
}}
static void Main(string[] args)
{
Console.WriteLine("enter the number");
long num = Convert.ToInt64(Console.ReadLine());
Console.WriteLine(new Program().fun(num));
Console.ReadLine();
}
}
}
hello everybody according to this solution i have my own solution where i calculate factorial of array 1D elements. the code is `int[] array = new int[5]
{
4,3,4,3,8
};
int fac = 1;
int[] facs = new int[array.Length+1];
for (int i = 0; i < array.Length; i++)
{
for (int j = array[i]; j > 0; j--)
{
fac *= j;
}
facs[i] = fac;
textBox1.Text += facs[i].ToString() + " ";
fac = 1;
}`
copy and paste the code above ^ in the button , it solves factorial of elements of array 1D. best regards.