We Keep Coding

My neural networks works well only with no hidden layers

You may think I am crazy, but I decided to write a neural network from scratch in C# for studying purposes. Please, be patient, I still have little experience)). English is not my first language, so I am sorry for it in advance.
I started with a program for handwritten digit recognizing with the MNIST database. I've read through a book about the algorithms inside the process and wrote this code.
public class NeuralNetwork
{
private List<Matrix<double>> weights = new List<Matrix<double>>();
private List<Vector<double>> biases = new List<Vector<double>>();
private Random random = new Random();
private List<Image> test_data;
private int[] layer_sizes;
public NeuralNetwork(params int[] layers)
{
layer_sizes = layers;
for (int i = 0; i < layers.Length - 1; i++)
{
var weigthLayer = Matrix<double>.Build.Dense(layers[i + 1], layers[i], (k, j) => random.NextDouble());
weights.Add(weigthLayer);
}
for (int i = 1; i < layers.Length; i++)
{
var biasesLayer = Vector<double>.Build.Dense(layers[i], (k) => random.NextDouble());
biases.Add(biasesLayer);
}
}
public Vector<double> FeedForward(Vector<double> a)
{
for (int i = 0; i < weights.Count; i++)
{
a = Sigmoid(weights[i].Multiply(a) + biases[i]);
}
return Sigmoid(a);
}
public void SGD(ITrainingDataProvider dataProvider, int epochs, int chuck_size, double eta)
{
test_data = new MnistReader().ReadTestData();
Console.WriteLine("SGD algorithm started");
var training_data = dataProvider.ReadTrainingData();
Console.WriteLine("Training data has beeen read");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
for (int epoch = 0; epoch < epochs; epoch++)
{
training_data = training_data.OrderBy(item => random.Next()).ToList();
List<List<Image>> chunks = training_data.ChunkBy(chuck_size);
foreach (List<Image> chunk in chunks)
{
ProcessChunk(chunk, eta);
}
Console.WriteLine($"Epoch: {epoch + 1}/{epochs}");
Console.WriteLine($"Training data test: {Test(training_data)}%");
Console.WriteLine($"Test data test: {Test(test_data)}%");
}
Console.WriteLine("Done!");
}
private double Test(List<Image> data)
{
int count = 0;
foreach (Image im in data)
{
var output = FeedForward(im.DataToVector());
int number = output.MaximumIndex();
if (number == (int)im.Label)
{
count++;
}
}
return (double)count / data.Count * 100;
}
private void ProcessChunk(List<Image> chunk, double eta)
{
Delta[] deltas = new Delta[chunk.Count];
for (int i = 0; i < chunk.Count; i++)
{
Image image = chunk[i];
var input = image.DataToVector();
var desired_output = Vector<double>.Build.Dense(layer_sizes[layer_sizes.Length - 1]);
desired_output[(int)image.Label] = 1;
Delta delta = BackPropagation(input, desired_output);
deltas[i] = delta;
}
Delta sum = deltas[0];
for (int i = 1; i < deltas.Length; i++)
{
sum += deltas[i];
}
Delta average_delta = sum / deltas.Length;
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
weights[i] += average_delta.d_weights[i].Multiply(eta);
biases[i] += average_delta.d_biases[i].Multiply(eta);
}
}
private Delta BackPropagation(Vector<double> input, Vector<double> desired_output)
{
List<Vector<double>> activations = new List<Vector<double>>();
List<Vector<double>> zs = new List<Vector<double>>();
Vector<double> a = input;
activations.Add(input);
for (int i = 0; i < layer_sizes.Length - 1; i++)
{
var z = weights[i].Multiply(a) + biases[i];
zs.Add(z);
a = Sigmoid(z);
activations.Add(a);
}
List<Vector<double>> errors = new List<Vector<double>>();
List<Matrix<double>> delta_weights = new List<Matrix<double>>();
List<Vector<double>> delta_biases = new List<Vector<double>>();
var error = CDerivative(activations[activations.Count - 1], desired_output).HProd(SigmoidDerivative(zs[^1]));
errors.Add(error);
int steps = 0;
for (int i = layer_sizes.Length - 2; i >= 1; i--)
{
var layer_error = weights[i].Transpose().Multiply(errors[steps]).HProd(SigmoidDerivative(zs[i - 1]));
errors.Add(layer_error);
steps++;
}
errors.Reverse();
for (int i = layer_sizes.Length - 1; i >= 1; i--)
{
var delta_layer_weights = (errors[i - 1].ToColumnMatrix() * activations[i - 1].ToRowMatrix()).Multiply(-1);
delta_weights.Add(delta_layer_weights);
var delta_layer_biases = errors[i - 1].Multiply(-1);
delta_biases.Add(delta_layer_biases);
}
delta_biases.Reverse();
delta_weights.Reverse();
return new Delta { d_weights = delta_weights, d_biases = delta_biases };
}
private Vector<double> CDerivative(Vector<double> x, Vector<double> y)
{
return x - y;
}
private Vector<double> Sigmoid(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = 1.0 / (1.0 + Math.Exp(-x[i]));
}
return x;
}
private Vector<double> SigmoidDerivative(Vector<double> x)
{
for (int i = 0; i < x.Count; i++)
{
x[i] = Math.Exp(-x[i]) / Math.Pow(1.0 + Math.Exp(-x[i]), 2);
}
return x;
}
}
Delta class. A simple DTO to store weights and biases changes in a single object.
public class Delta
{
public List<Matrix<double>> d_weights { get; set; }
public List<Vector<double>> d_biases { get; set; }
public static Delta operator +(Delta d1, Delta d2)
{
Delta result = d1;
for (int i = 0; i < d2.d_weights.Count; i++)
{
result.d_weights[i] += d2.d_weights[i];
}
for (int i = 0; i < d2.d_biases.Count; i++)
{
result.d_biases[i] += d2.d_biases[i];
}
return result;
}
public static Delta operator /(Delta d1, double d)
{
Delta result = d1;
for (int i = 0; i < d1.d_weights.Count; i++)
{
result.d_weights[i] /= d;
}
for (int i = 0; i < d1.d_biases.Count; i++)
{
result.d_biases[i] /= d;
}
return result;
}
}
Everything ended up working fine, however complex networks with 1 or more hidden layers don't show any significant results. They are getting best 70% accuracy and then the learning curve drops. The accuracy returns to its 20-30%. Typically, the graph looks like a square root function, but in my case it is more like a turned around quadratic parabola the graph of my tries with different amounts of neurons in the first hidden layer
After a few tries, I found out, that without any hidden layers the algorithm works just fine. It learns up to 90% of accuracy and then the graph never falls down. Apparently, the bug is somewhere in back-propagation algorithm. It doesn't cause any problems with only input and output layers, but it does, when I add a hidden layer.
I have been trying to find the problem for a long time and I hope that someone, smarter than me, will be able to help.
Thanks in advance!

Can somebody tell me why this code is not working? I am trying to find the 10001st prime number

The code gives me the answer 43739 which is wrong. I don't know which part of the code I messed up, help would be much appreciated.
{
int primenumbers = 4;
int thenumber = 2;
int finalnumber = 0;
while (primenumbers <= 10001)
{
for (int x = 2; x < 10; x++)
{
if (thenumber % x == 0)
{
x = 10;
}
if (x == 9)
{
finalnumber = thenumber;
primenumbers += 1;
break;
}
}
thenumber += 1;
}
Console.WriteLine(finalnumber);
}

Let's split the initial problem into smaller ones: first, check for being prime:
private static bool IsPrime(int value) {
if (value <= 1)
return false;
else if (value % 2 == 0)
return value == 2;
int n = (int) (Math.Sqrt(value) + 0.5);
for (int d = 3; d <= n; d += 2)
if (value % d == 0)
return false;
return true;
}
Then enumerate prime numbers (2, 3, 5, 7, 11...):
// Naive; sieve algorithm will be better off
private static IEnumerable<int> Primes() {
for (int i = 1; ; ++i)
if (IsPrime(i))
yield return i;
}
Finally, query with a help of Linq:
using System.Linq;
...
int result = Primes().Skip(10000).First();
Console.Write(result);
And you'll get
104743

Unable to detect if two rectangles are colliding C#

I am working on a procedural room generator in c#, I would like the rooms not to overlap and I am having a hard time getting that to work. After #Idle_Mind's comments, I have a new problem. The Image produced by the program has many overlapping rooms. Bellow is the class that should handle the intersection checking and the placement of the rooms onto the tilemap
public int XSize, YSize;
private Cell[ , ] cells;
private List<Room> rooms;
public Tilemap(int xSize, int ySize)
{
XSize = xSize;
YSize = ySize;
rooms = new List<Room>();
cells = new Cell[XSize, YSize];
for (int y = 0; y < YSize; y++)
{
for (int x = 0; x < XSize; x++)
{
cells[x, y].type = CellType.Empty;
}
}
for (int i = 0; i < 10; i++)
{
GenerateRandomRoomSafe(10);
}
}
private Room GetRoomBounds()
{
Utils.Int2 min = new Utils.Int2(0, 0);
Utils.Int2 max = new Utils.Int2(XSize,YSize);
Utils.Int2 q1 = Utils.GetRandomCoords(min, max);
max.X = XSize - 1 - q1.X;
max.Y = YSize - 1 - q1.Y;
Utils.Int2 siz = Utils.GetRandomCoords(min, max);
Room check = new Room(q1.X, q1.Y, siz.X, siz.Y);
return check;
}
public void GenerateRandomRoomSafe(int maxTries)
{
Room check = new Room(0, 0, 0, 0);
bool isValid = false;
int tries = 0;
if (rooms.Count == 0)
{
isValid = true;
check = GetRoomBounds();
tries = 1;
}
else
{
while (!isValid && tries < maxTries)
{
check = GetRoomBounds();
for (int i = 0; i < rooms.Count; i++)
{
if (!rooms[i].Walls.IntersectsWith(check.Walls))
{
isValid = true;
break;
}
}
tries++;
}
}
if (isValid)
{
Console.WriteLine(check + " was placed after " + tries + " tries");
PlaceRoomUnsafe(check);
}
}
public void PlaceRoomUnsafe(Room r)
{
for (int y = r.Walls.Y; y <= r.Walls.Y + r.Walls.Height; y++)
{
cells[r.Walls.X, y].type = CellType.Wall;
}
for (int y = r.Walls.Y; y <= r.Walls.Y + r.Walls.Height; y++)
{
cells[r.Walls.X + r.Walls.Width, y].type = CellType.Wall;
}
for (int x = r.Walls.X; x <= r.Walls.X + r.Walls.Width; x++)
{
cells[x, r.Walls.Y].type = CellType.Wall;
}
for (int x = r.Walls.X; x <= r.Walls.X + r.Walls.Width; x++)
{
cells[x, r.Walls.Y + r.Walls.Height].type = CellType.Wall;
}
for (int y = r.Floor.Y; y < r.Floor.Y + r.Floor.Height; y++)
{
for (int x = r.Floor.X; x < r.Floor.X + r.Floor.Width; x++)
{
cells[x, y].type = CellType.Floor;
}
}
rooms.Add(r);
}
public void GenerateRandomRoomUnsafe()
{
Room r = GetRoomBounds();
PlaceRoomUnsafe(r);
}
public int GetCellPixel(int x, int y)
{
return (int) cells[x, y].type;
}
public class Room
{
public Rectangle Walls;
public Rectangle Floor;
public Room(int q1X, int q1Y, int xSize, int ySize)
{
Walls.X = q1X;
Walls.Y = q1Y;
Walls.Width = xSize;
Walls.Height = ySize;
Floor.X = q1X + 1;
Floor.Y = q1Y + 1;
Floor.Width = xSize - 1;
Floor.Height = ySize - 1;
}
public override string ToString()
{
return "[Walls: " + Walls + "\n Floor: " + Floor + "]";
}
}
}
Image below for reference

Your logic in GenerateRandomRoomSafe() is backwards.
In this code block:
while (!isValid && tries < maxTries)
{
check = GetRoomBounds();
for (int i = 0; i < rooms.Count; i++)
{
if (!rooms[i].Walls.IntersectsWith(check.Walls))
{
isValid = true;
break;
}
}
tries++;
}
What you're saying is, "If it doesn't intersect with at least one room, then it must be valid".
Just because it doesn't intersect with the current room, doesn't mean it won't intersect with a different one!
It should look more like: (note the comments)
while (!isValid && tries < maxTries)
{
isValid = true; // assume it's good until proven otherwise
check = GetRoomBounds();
for (int i = 0; i < rooms.Count; i++)
{
if (rooms[i].Walls.IntersectsWith(check.Walls)) // if it DOES intersect...
{
isValid = false; // .. then set valid to false.
break;
}
}
tries++;
}
So we start by assuming it is valid, then when we encounter a room that DOES intersect, we set valid to false and stop the for loop so another random room can be tried (assuming we haven't exceeded the number of tries).
Some of the maps it produced:

Implementing and using MinMax with four in row (connect4) game

I'm trying to implement the MinMax algorithm for four in a row (or connect4 or connect four) game.
I think I got the idea of it, it should build a tree of possible boards up to a certain depth, evaluate them and return their score, then we just take the max of those scores.
So, aiChooseCol() checks the score of every possible column by calling MinMax() and returns the column with the max score.
Now I wasn't sure, is this the right way to call MinMax()?
Is it right to check temp = Math.Max(temp, 1000);?
I still haven't made the heuristic function but this should at least recognize a winning column and choose it, but currently it just choose the first free column from the left... I can't figure out what am I doing wrong.
private int AiChooseCol()
{
int best = -1000;
int col=0;
for (int i = 0; i < m_Board.Cols; i++)
{
if (m_Board.CheckIfColHasRoom(i))
{
m_Board.FillSignInBoardAccordingToCol(i, m_Sign);
int t = MinMax(5, m_Board, board.GetOtherPlayerSign(m_Sign));
if (t > best)
{
best = t;
col = i;
}
m_Board.RemoveTopCoinFromCol(i);
}
}
return col;
}
private int MinMax(int Depth, board Board, char PlayerSign)
{
int temp=0;
if (Depth <= 0)
{
// return from heurisitic function
return temp;
}
char otherPlayerSign = board.GetOtherPlayerSign(PlayerSign);
char checkBoard = Board.CheckBoardForWin();
if (checkBoard == PlayerSign)
{
return 1000;
}
else if (checkBoard == otherPlayerSign)
{
return -1000;
}
else if (!Board.CheckIfBoardIsNotFull())
{
return 0; // tie
}
if (PlayerSign == m_Sign) // maximizing Player is myself
{
temp = -1000;
for (int i = 0; i < Board.Cols; i++)
{
if (Board.FillSignInBoardAccordingToCol(i, PlayerSign)) // so we don't open another branch in a full column
{
var v = MinMax(Depth - 1, Board, otherPlayerSign);
temp = Math.Max(temp, v);
Board.RemoveTopCoinFromCol(i);
}
}
}
else
{
temp = 1000;
for (int i = 0; i < Board.Cols; i++)
{
if (Board.FillSignInBoardAccordingToCol(i, PlayerSign)) // so we don't open another branch in a full column
{
var v = MinMax(Depth - 1, Board, otherPlayerSign);
temp = Math.Min(temp, v);
Board.RemoveTopCoinFromCol(i);
}
}
}
return temp;
}
Some notes:
FillSignInBoardAccordingToCol() returns a boolean if it was successful.
The board type has a char[,] array with the actual board and signs of the players.
This code is in the AI Player class.

So I decided to write my own MinMax Connect 4. I used the depth to determine the value of a win or loss so that a move that gets you closer to winning or blocking a loss will take precedence. I also decide that I will randomly pick the move if more than one has the same heuristic. Finally I stretched out the depth to 6 as that's how many moves are required to find possible win paths from the start.
private static void Main(string[] args)
{
var board = new Board(8,7);
var random = new Random();
while (true)
{
Console.WriteLine("Pick a column 1 -8");
int move;
if (!int.TryParse(Console.ReadLine(), out move) || move < 1 || move > 8)
{
Console.WriteLine("Must enter a number 1-8.");
continue;
}
if (!board.DropCoin(1, move-1))
{
Console.WriteLine("That column is full, pick another one");
continue;
}
if (board.Winner == 1)
{
Console.WriteLine(board);
Console.WriteLine("You win!");
break;
}
if (board.IsFull)
{
Console.WriteLine(board);
Console.WriteLine("Tie!");
break;
}
var moves = new List<Tuple<int, int>>();
for (int i = 0; i < board.Columns; i++)
{
if (!board.DropCoin(2, i))
continue;
moves.Add(Tuple.Create(i, MinMax(6, board, false)));
board.RemoveTopCoin(i);
}
int maxMoveScore = moves.Max(t => t.Item2);
var bestMoves = moves.Where(t => t.Item2 == maxMoveScore).ToList();
board.DropCoin(2, bestMoves[random.Next(0,bestMoves.Count)].Item1);
Console.WriteLine(board);
if (board.Winner == 2)
{
Console.WriteLine("You lost!");
break;
}
if (board.IsFull)
{
Console.WriteLine("Tie!");
break;
}
}
Console.WriteLine("DONE");
Console.ReadKey();
}
private static int MinMax(int depth, Board board, bool maximizingPlayer)
{
if (depth <= 0)
return 0;
var winner = board.Winner;
if (winner == 2)
return depth;
if (winner == 1)
return -depth;
if (board.IsFull)
return 0;
int bestValue = maximizingPlayer ? -1 : 1;
for (int i = 0; i < board.Columns; i++)
{
if (!board.DropCoin(maximizingPlayer ? 2 : 1, i))
continue;
int v = MinMax(depth - 1, board, !maximizingPlayer);
bestValue = maximizingPlayer ? Math.Max(bestValue, v) : Math.Min(bestValue, v);
board.RemoveTopCoin(i);
}
return bestValue;
}
public class Board
{
private readonly int?[,] _board;
private int? _winner;
private bool _changed;
public Board(int cols, int rows)
{
Columns = cols;
Rows = rows;
_board = new int?[cols, rows];
}
public int Columns { get; }
public int Rows { get; }
public bool ColumnFree(int column)
{
return !_board[column, 0].HasValue;
}
public bool DropCoin(int playerId, int column)
{
int row = 0;
while (row < Rows && !_board[column,row].HasValue)
{
row++;
}
if (row == 0)
return false;
_board[column, row - 1] = playerId;
_changed = true;
return true;
}
public bool RemoveTopCoin(int column)
{
int row = 0;
while (row < Rows && !_board[column, row].HasValue)
{
row++;
}
if (row == Rows)
return false;
_board[column, row] = null;
_changed = true;
return true;
}
public int? Winner
{
get
{
if (!_changed)
return _winner;
_changed = false;
for (int i = 0; i < Columns; i++)
{
for (int j = 0; j < Rows; j++)
{
if (!_board[i, j].HasValue)
continue;
bool horizontal = i + 3 < Columns;
bool vertical = j + 3 < Rows;
if (!horizontal && !vertical)
continue;
bool forwardDiagonal = horizontal && vertical;
bool backwardDiagonal = vertical && i - 3 >= 0;
for (int k = 1; k < 4; k++)
{
horizontal = horizontal && _board[i, j] == _board[i + k, j];
vertical = vertical && _board[i, j] == _board[i, j + k];
forwardDiagonal = forwardDiagonal && _board[i, j] == _board[i + k, j + k];
backwardDiagonal = backwardDiagonal && _board[i, j] == _board[i - k, j + k];
if (!horizontal && !vertical && !forwardDiagonal && !backwardDiagonal)
break;
}
if (horizontal || vertical || forwardDiagonal || backwardDiagonal)
{
_winner = _board[i, j];
return _winner;
}
}
}
_winner = null;
return _winner;
}
}
public bool IsFull
{
get
{
for (int i = 0; i < Columns; i++)
{
if (!_board[i, 0].HasValue)
return false;
}
return true;
}
}
public override string ToString()
{
var builder = new StringBuilder();
for (int j = 0; j < Rows; j++)
{
builder.Append('|');
for (int i = 0; i < Columns; i++)
{
builder.Append(_board[i, j].HasValue ? _board[i,j].Value.ToString() : " ").Append('|');
}
builder.AppendLine();
}
return builder.ToString();
}
}

The knights tour for 8x8 infinite recurssion

here will be my code for famous Knights Tour for 8x8 deck. So, the main idea of my code is: we will choose from Turns our destination, check it with isPossible and then go recursevly to it, marked this cell to '1'. So, check every cell, and if we will be in 64's cell - return true.
But my code goes to infinite recurssion, and I can't debug it, any recommendation will be greatly appreciated.
class Class1
{
static void Main(string[] args)
{
int x = 0;
int y = 0;
Console.WriteLine("Enter X and press enter");
x = Int32.Parse(Console.ReadLine());
Console.WriteLine("Enter Y and press enter");
y = Int32.Parse(Console.ReadLine());
TurnVariation Turns = new TurnVariation();
EmptyBoard Board = new EmptyBoard();
if (TryPut.Put(Board, x, y, Turns, 1, false))
{
Console.WriteLine("МОЖНА!!!!");
}
else
{
Console.WriteLine("NET!!");
}
}
}
public class TryPut : EmptyBoard
{
public static bool Put(EmptyBoard Board, int x, int y, TurnVariation Turns, int count, bool flag)
{
int tempX = 0;
int tempY = 0;
if (count >= 64)
{
Console.WriteLine("yeab");
return true;
}
for (int i = 0; i <= 7; i++)
{
tempX = x + Turns.Turns[i,0];
tempY = y + Turns.Turns[i,1];
//Console.WriteLine(count);
if (IsPossible(Board, tempX, tempY))
{
Board.Array[tempX, tempY] = 1;
flag = Put(Board, tempX, tempY, Turns, count+1, flag);
if (flag)
{
break;
}
Board.Array[tempX, tempY] = 0;
}
}
if (flag)
return true;
else
return false;
}
public static bool IsPossible(EmptyBoard Board, int x, int y)
{
if ((x < 0) || (x > 7) || (y < 0) || (y > 7))
return false;
if (Board.Array[x, y] == 1)
return false;
return true;
}
}
public class TurnVariation
{
public int[,] Turns = new int[8, 2];
public TurnVariation()
{
Turns[0, 0] = -2; Turns[0, 1] = 1;
Turns[1,0] = -2; Turns[1,1] = -1;
Turns[2,0] = -1; Turns[2,1] = 2;
Turns[3,0] = 1; Turns[3,1] = 2;
Turns[4,0] = 2; Turns[4,1] = 1;
Turns[5,0] = 2; Turns[5,1] = -1;
Turns[6,0] = 1; Turns[6,1] = -2;
Turns[7,0] = -1; Turns[7,1] = -2;
}
}
public class EmptyBoard
{
public const int N = 8;
public int[,] Array = new int[N, N];
public EmptyBoard()
{
for (int i = 0; i < N; i++)
for (int j = 0; j < N; j++)
Array[i, j] = 0;
}
}

I think your problem is that your testing for count<64, but you never assign to count. You just pass (by value!) 'Count + 1' to the put method. You are probably thinking that this will bewritten back to the count variable. But that is not the case...
Do note that debugging is the first skill you need to learn!