I am implementing Yolov4-Tiny (onnx model found here) in Unity with the Windows ML APIs. I can load the model, and begin a session with no issue. I am using a VideoFrame (sized to 416x416) as the input and can access the two output Tensors. The problems arise when I begin to parse the output Tensors. With a confidence threshold of .5, I get between 700 and 1000 detections each frame, way way more than I expect. Also, the bboxes appear to be very small. The NMS and IOU functions below are nearly verbatim from here so I am not using anchors for the bounding boxes. I believe my issues is in the NMS and IOU functions, but I cannot locate the problem. My gut tells me I am manipulating the output Tensors incorrectly. Any ideas?
private List<DetectionResult> ParseResult(float[] boxes, float[] classes)
{
int c_values = 80;
int c_boxes = boxes.Length / 4;
int c_classNames = classes.Length / c_values;
float confidence_threshold = 0.5f;
List<DetectionResult> detections = new List<DetectionResult>();
for (int i_box = 0; i_box < c_classNames; i_box++)
{
float max_prob = 0.0f;
int label_index = -1;
for (int j_confidence = 0; j_confidence < c_values; j_confidence++)
{
int index = i_box * c_values + j_confidence;
if (Sigmoid(classes[index]) > max_prob)
{
max_prob = Sigmoid(classes[index]) ;
label_index = j_confidence;
}
}
if (max_prob > confidence_threshold)
{
//Debug.Log(_labels[label_index]);
List<float> bbox = new List<float>();
bbox.Add(boxes[(i_box * 4) + 0] * 416);
bbox.Add(boxes[(i_box * 4) + 1] * 416);
bbox.Add(boxes[(i_box * 4) + 2] * 416);
bbox.Add(boxes[(i_box * 4) + 3] * 416);
detections.Add(new DetectionResult()
{
label = _labels[label_index],
bbox = bbox,
prob = max_prob
});
}
}
private List<DetectionResult> NMS(IReadOnlyList<DetectionResult> detections,
float IOU_threshold = 0.45f,
float score_threshold = 0.3f)
{
List<DetectionResult> final_detections = new List<DetectionResult>();
for (int i = 0; i < detections.Count; i++)
{
int j = 0;
for (j = 0; j < final_detections.Count; j++)
{
if (ComputeIOU(final_detections[j], detections[i]) > IOU_threshold)
{
break;
}
}
if (j == final_detections.Count)
{
final_detections.Add(detections[i]);
}
}
return final_detections;
}
private float ComputeIOU(DetectionResult DRa, DetectionResult DRb)
{
float ay1 = DRa.bbox[0];
float ax1 = DRa.bbox[1];
float ay2 = DRa.bbox[2];
float ax2 = DRa.bbox[3];
float by1 = DRb.bbox[0];
float bx1 = DRb.bbox[1];
float by2 = DRb.bbox[2];
float bx2 = DRb.bbox[3];
// determine the coordinates of the intersection rectangle
float x_left = Math.Max(ax1, bx1);
float y_top = Math.Max(ay1, by1);
float x_right = Math.Min(ax2, bx2);
float y_bottom = Math.Min(ay2, by2);
if (x_right < x_left || y_bottom < y_top)
return 0;
float intersection_area = (x_right - x_left) * (y_bottom - y_top);
float bb1_area = (ax2 - ax1) * (ay2 - ay1);
float bb2_area = (bx2 - bx1) * (by2 - by1);
float iou = intersection_area / (bb1_area + bb2_area - intersection_area);
return iou;
}
The problem
I am trying to procedurally generate dungeon rooms with random X, Y sizes inside of a radius (r). However, even after I validate that the starting grid (origin of the "room") is not in the same position as other origins after running the separation function there are rooms still building inside of each other.
Solutions I have tried
I tried using math to calculate an optimal radius that will be able to fit the average of all the room sizes * amount of rooms. However, the separation should hypothetically work with any radius (though I want to keep them relatively close in order to keep hallways short).
Code
All my code is based on one tile. This means that all calculations are using one tile, and will remain one tile until the very end, then I scale them up.
private void GenerateRooms(int amount)
{
// init sizes
Vector2[] room_sizes = new Vector2[amount];
for (int i = 0; i < amount; i++)
{
room_sizes[i] = new Vector2(Random.Range(minimum_room_height, maximum_room_height), Random.Range(minimum_room_width, maximum_room_width));
}
float biggest_room = calculations.CalculateBiggest(room_sizes);
Vector2[] room_points = new Vector2[amount];
Vector2[] used_points = new Vector2[amount];
float radius = calculations.CalculateAverage(room_sizes) * amount;
for (int i = 0; i < amount; i++)
{
do {
Vector2 test_point = new Vector2(Random.Range(-radius, radius), Random.Range(-radius, radius));
foreach (Vector2 point in used_points) {
if (test_point == point) {
continue;
} else {
room_points[i] = test_point;
used_points[i] = test_point;
break;
}
}
} while (Vector2.Distance(Vector2.zero, room_points[i]) < radius);
}
for (int i = 0; i < amount; i++)
{
//Vector2 origin = room_points[i];
Vector3 position = calculations.computeSeperate(room_points, room_points[i], biggest_room);
//position = new Vector3(position.x + origin.x, position.y + origin.y, 0);
Vector3Int location = tile_map.WorldToCell(position);
tile_map.SetTile(location, tile);
calculations.scaleUpRooms(position, room_sizes[i].x, room_sizes[i].y, tile_map, tile);
}
}
The above is code for calling all the functions and validating the points. Here are the important functions (calculation functions):
public Vector2 computeSeperate(Vector2[] point_array, Vector2 target_point, float minimum_distance)
{
int neighbor_count = 0;
for (int i = 0; i < point_array.Length; i++)
{
if (point_array[i] != target_point)
{
if (Vector2.Distance(target_point, point_array[i]) < minimum_distance * 2)
{
target_point.x += point_array[i].x - target_point.x;
target_point.y += point_array[i].y - target_point.y;
neighbor_count++;
}
}
}
if (neighbor_count == 0)
{
return target_point;
} else
{
target_point.x /= neighbor_count;
target_point.y /= neighbor_count;
target_point.x *= -1;
target_point.y *= -1;
target_point.Normalize();
return target_point;
}
}
public void scaleUpRooms(Vector2 base_point, float scale_x, float scale_y, Tilemap tile_map, Tile tile) // ex: 5x5
{
List<Vector2> Calculate(Vector2 size)
{
List<Vector2> results = new List<Vector2>();
for (int i = 0; i < size.y; i++)
for (int o = 0; o < size.x; o++)
results.Add(new Vector2(o, i) + (new Vector2(size.x % 2 != 0 ? .5f : 1, size.y % 2 != 0 ? .5f : 1) - (size / 2)));
string st = "";
for (int i = 0; i < results.Count; i++)
st += "\n" + results[i].ToString();
return results;
}
Vector2 desired_scale = new Vector2(scale_x, scale_y);
List<Vector2> Offsets = Calculate(desired_scale);
for (int i = 0; i < Offsets.Count; i++)
{
Vector3 position = base_point + Offsets[i];
Vector3Int location = tile_map.WorldToCell(position);
tile_map.SetTile(location, tile);
}
}
I have a problem with my terrain.
When I want to set the altitude it works, but the terrain is always displayed larger than the 2D array I use for the altitude. In this example, my array and terrain are 200x157. So why do I have this overhang in my display?
using MoonSharp.Interpreter;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
using System;
using System.Text;
using System.IO;
[MoonSharpUserData]
public class PrefabCreator : MonoBehaviour
{
/// <summary>
/// Classmembers for PrefabCreator
/// </summary>
private int terrainDepth;
private int terrainWidth;
private int terrainHeight;
private float terrainScale;
private float maxHeight;
public void CreateTerrain(int[][] mapData)
{
MapScale(mapData);
Terrain terrain = Resources.Load<Terrain>("TerrainObject") as Terrain;
Terrain prefabTerrain = Instantiate(terrain, new Vector3(0, 0, 0), Quaternion.identity);
prefabTerrain.terrainData = GenerateTerrain(prefabTerrain.terrainData, mapData);
}
private void MapScale(int[][] mapData)
{
terrainWidth = 0;
terrainHeight = 0;
terrainDepth = 10;
maxHeight = 0;
bool firstRun = true;
for (int i = 0; i < mapData.GetLength(0); i++)
{
terrainHeight++;
for (int j = 0; j < mapData[i].Length; j++)
{
if(mapData[i][j] > maxHeight)
{
maxHeight = mapData[i][j];
}
if (firstRun)
{
terrainWidth++;
}
}
firstRun = false;
}
Debug.Log("MAXHEIGHT: " + maxHeight);
}
private TerrainData GenerateTerrain(TerrainData terrainData, int[][] mapData)
{
if(terrainWidth >= terrainHeight)
{
terrainData.heightmapResolution = terrainWidth + 1;
}
else
{
terrainData.heightmapResolution = terrainHeight + 1;
}
terrainData.size = new Vector3(terrainWidth, terrainDepth, terrainHeight);
Debug.Log("Test" + terrainData.heightmapHeight);
Debug.Log("Test" + terrainData.heightmapWidth);
Debug.Log("Test" + terrainData.heightmapScale);
terrainData.SetHeights(0, 0, GenerateHeights(mapData));
return terrainData;
}
private float[,] GenerateHeights(int[][] mapData)
{
Debug.Log("Breite " + terrainWidth);
Debug.Log("Höhe " + terrainHeight);
float[,] heights = new float[terrainHeight, terrainWidth];
//Debug.Log("Test" + mapData[2][4]);
for (int x = 0; x <= terrainHeight - 1; x++)
{
//Debug.Log("X = " + x);
for (int y = 0; y <= terrainWidth - 1; y++)
{
//Debug.Log("Y = " + y);
float value = 0.0f + mapData[x][y];
heights[x, y] = (value / maxHeight);
//heights[x, y] = (float)value ;
Debug.Log(heights[x, y]);
}
}
return heights;
}
}
See the picture in the appendix. I don't understand why the overhangs exist at the edges?
Many thanks in advance People
I'm building a tile engine for my first game ever and i've been following a guide step by step (can't link it because i'm limited to 2). I have however, made a few modifications to the tilemap so that i can continue with my project.
And while testing my program a noticed a bug that i was hoping you guys could help me resolve...
...The tile engine is drawing the wrong tiles!
This is my TileMap (with numbers assigned to each tile) and this is what my engine draws.
While it should follow the order as seen below (the code is self-explanatory)
rows[0].columns[3].tileID = 0;
rows[0].columns[4].tileID = 1;
rows[0].columns[5].tileID = 2;
rows[0].columns[6].tileID = 3;
rows[0].columns[7].tileID = 4;
rows[1].columns[3].tileID = 5;
rows[1].columns[4].tileID = 6;
rows[1].columns[5].tileID = 7;
rows[1].columns[6].tileID = 8;
rows[1].columns[7].tileID = 9;
rows[2].columns[3].tileID = 10;
rows[2].columns[4].tileID = 11;
rows[2].columns[5].tileID = 12;
rows[2].columns[6].tileID = 13;
rows[2].columns[7].tileID = 14;
rows[3].columns[3].tileID = 15;
rows[3].columns[4].tileID = 16;
rows[3].columns[5].tileID = 17;
rows[3].columns[6].tileID = 18;
rows[3].columns[7].tileID = 19;
rows[4].columns[3].tileID = 20;
rows[4].columns[4].tileID = 21;
rows[4].columns[5].tileID = 22;
rows[4].columns[6].tileID = 23;
rows[4].columns[7].tileID = 24;
rows[5].columns[3].tileID = 25;
rows[5].columns[4].tileID = 26;
rows[5].columns[5].tileID = 27;
rows[5].columns[6].tileID = 28;
rows[5].columns[7].tileID = 29;
Code Essentials
The Important parts of my code
Tile.class
Where i define the tile size and have a function which navigates the tilemap to find my deiserd tile.
namespace TileEngine
{
static class Tile
{
static public Texture2D tileSetTexture;
static public int tileWidth = 48;
static public int tileHeight = 48;
static public Rectangle getSourceRectangle(int tileIndex)
{
int tileY = tileIndex / (tileSetTexture.Width / tileWidth);
int tileX = tileIndex % (tileSetTexture.Height / tileHeight);
return new Rectangle(tileX * tileWidth, tileY * tileHeight, tileWidth, tileHeight);
}
}
}
MapCell
Where i define the tileID
class MapCell
{
public List<int> baseTiles = new List<int>();
public int tileID
{
get { return baseTiles.Count > 0 ? baseTiles[0] : 0; }
set
{
if (baseTiles.Count > 0)
baseTiles[0] = value;
else
addBaseTile(value);
}
}
public void addBaseTile(int tileID)
{
baseTiles.Add(tileID);
}
public MapCell(int tileID)
{
this.tileID = tileID;
}
}
TileMap
Where i build the map
class MapRow
{
public List<MapCell> columns = new List<MapCell>();
}
class TileMap
{
public List<MapRow> rows = new List<MapRow>();
public int mapWidth = 50;
public int mapHeight = 50;
public TileMap()
{
for (int y = 0; y < mapHeight; y++)
{
MapRow thisRow = new MapRow();
for (int x = 0; x < mapWidth; x++)
{
thisRow.columns.Add(new MapCell(0));
}
rows.Add(thisRow);
}
}
}
Draw
And lastly my draw code.
protected override void Draw(GameTime gameTime)
{
GraphicsDevice.Clear(Color.Black);
spriteBatch.Begin();
Vector2 firstSquare = new Vector2(Camera.location.X / Tile.tileWidth, Camera.location.Y / Tile.tileHeight);
int firstX = (int)firstSquare.X;
int firstY = (int)firstSquare.Y;
Vector2 squareOffset = new Vector2(Camera.location.X % Tile.tileWidth, Camera.location.Y % Tile.tileHeight);
int offsetX = (int)squareOffset.X;
int offsetY = (int)squareOffset.Y;
for (int y = 0; y < squaresDown; y++)
{
for (int x = 0; x < squaresAcross; x++)
{
foreach (int tileID in myMap.rows[y + firstY].columns[x + firstX].baseTiles)
{
spriteBatch.Draw(
Tile.tileSetTexture,
new Rectangle(
(x * Tile.tileWidth) - offsetX, (y * Tile.tileHeight) - offsetY,
Tile.tileWidth, Tile.tileHeight),
Tile.getSourceRectangle(tileID),
Color.White);
}
}
}
spriteBatch.End();
base.Draw(gameTime);
}
I cannot for the life of me work out how to get the block that is supposed to be drawn with every loop through the array of "Arxl" objects to animate across the grid.
Any suggestions would be really appreciated, not looking for someone to complete the code for me. just a fresh set of eyes.
public partial class Game : Form
{
//attributes
private Bitmap _grid;
private Arxl[,] _cartesianGrid;
private int _arxlAmount;
const int ARXL = 4;
public Game()
{
InitializeComponent();
_arxlAmount = (gridPictureBox.Height / ARXL);//in case height/arxl is not an even number?
_cartesianGrid = new Arxl[_arxlAmount, _arxlAmount];
_grid = new Bitmap(gridPictureBox.Width, gridPictureBox.Height);
int x;
int y;
for (x = 0; x < _arxlAmount; x++)
{
for (y = 0; y < _arxlAmount; y++)
{
_cartesianGrid[x, y] = new Arxl();
}
}
SetSeed(_cartesianGrid);
}
private void SetSeed(Arxl[,] cartesianGrid)
{
_cartesianGrid[1, 1].Active = true;
}
private void DrawArxl(Bitmap _grid, Arxl[,] cartesianGrid,int arxlAmount)
{
int x, y;
x=0;
y=0;
Graphics graphics = Graphics.FromImage(_grid);
graphics.Clear(Color.White);
for (x = 1; x < arxlAmount;x++ )
{
for (y = 1; y < arxlAmount; y++)
{
if (cartesianGrid[x, y].Active==true)
{
cartesianGrid[x, y].Area = new Rectangle(x * ARXL, y * ARXL, ARXL, ARXL);
graphics.FillRectangle(Brushes.Black, cartesianGrid[x, y].Area);
}
else if(cartesianGrid[x,y].Active==false)
{
Pen newPen=new Pen(Color.Black);
cartesianGrid[x, y].Area = new Rectangle(x * ARXL, y * ARXL, ARXL, ARXL);
graphics.DrawRectangle(newPen,cartesianGrid[x, y].Area);
newPen.Dispose();
}
}
}
}
private void timer_Tick(object sender, EventArgs e)
{
//GameOfLife(_cartesianGrid, _arxlAmount);
ScrollBlock(_cartesianGrid, _arxlAmount);
DrawArxl(_grid, _cartesianGrid, _arxlAmount);
gridPictureBox.Image = _grid;
}
private void ScrollBlock(Arxl[,] cartesianGrid, int arxlAmount)
{
int x = 0;
int y = 0;
for (x = 0; x < arxlAmount; x++)
{
for (y = 0; y < arxlAmount; y++)
{
if (_cartesianGrid[x, y].Active == true)
{
if (x>=0)
{
if (x == (arxlAmount-1))
{
_cartesianGrid[x, y].Active = false;
_cartesianGrid[1, y].Active = true;
}
else if(x<(arxlAmount-1))
{
_cartesianGrid[x, y].Active = false;
_cartesianGrid[x+1, y].Active = true;
}
}
}
}
}
}
According to a comment in your code, you want to program the life game. It will not work, if you change the cells in place, because you will have to compute the new state from the unchanged old state. Therefore, you will need to have two game boards, one with the current state and one with the new state. Instead of creating new board all the time, it is better to have two boards and to swap them. In addition, there is no point in storing the Rectangles in the board. Therefore, I declare the boards as Boolean matrix.
const int CellSize = 4;
private int _boardSize;
private bool[,] _activeBoard, _inactiveBoard;
Bitmap _grid;
The form constructor is changed like this
public Game()
{
InitializeComponent();
_boardSize = Math.Min(gridPictureBox.Width, gridPictureBox.Height) / CellSize;
_grid = new Bitmap(gridPictureBox.Width, gridPictureBox.Height);
_activeBoard = new bool[_boardSize, _boardSize];
_inactiveBoard = new bool[_boardSize, _boardSize];
SetSeed();
}
We initialize the game like this (as an example)
private void SetSeed()
{
_activeBoard[0, 0] = true;
_activeBoard[7, 4] = true;
DrawGrid();
}
The timer tick does this
ScrollBlock();
DrawGrid();
The logic in ScrollBlock is completely new. We look at the state on the _activeBoard and set the state of _inactiveBoard. Then we swap the two boards.
private void ScrollBlock()
{
for (int x = 0; x < _boardSize; x++) {
for (int y = 0; y < _boardSize; y++) {
if (_activeBoard[x, y]) {
_activeBoard[x, y] = false;
int newX = x + 1;
int newY = y;
if (newX == _boardSize) {
newX = 0;
newY = (newY + 1) % _boardSize;
}
_inactiveBoard[newX, newY] = true;
}
}
}
SwapBoards();
}
The boards are simply swapped like this
private void SwapBoards()
{
bool[,] tmp = _activeBoard;
_activeBoard = _inactiveBoard;
_inactiveBoard = tmp;
}
And finally DrawGrid draws the _activeBoard
private void DrawGrid()
{
Graphics graphics = Graphics.FromImage(_grid);
graphics.Clear(Color.White);
for (int x = 0; x < _boardSize; x++) {
for (int y = 0; y < _boardSize; y++) {
var rect = new Rectangle(x * CellSize, y * CellSize, CellSize, CellSize);
if (_activeBoard[x, y]) {
graphics.FillRectangle(Brushes.Black, rect);
} else {
graphics.DrawRectangle(Pens.Black, rect);
}
}
}
gridPictureBox.Image = _grid;
}
I've spotted your problem.
The problem is that you're updating a cell position (moving it to the right in this particular initial state), but the next iteration in the for loop finds the updated state from the previous iteration, so it updates the cell again, and again, and when the cycle stops, the cell was scrolled over to its initial cell position!, with no repainting in between.
I'm modifying your code to add an UpdateList that will turn on cells that need to be ON after the grid scan has finished to avoid updating the same "active dot" more than once. This should show a moving dot from left to right.
private void ScrollBlock(Arxl[,] cartesianGrid, int arxlAmount) {
int x = 0;
int y = 0;
List<Point> updateList = new List<Point>();
for( x = 0; x < arxlAmount; x++ ) {
for( y = 0; y < arxlAmount; y++ ) {
if( _cartesianGrid[x, y].Active == true ) {
if( x >= 0 ) {
if( x == (arxlAmount - 1) ) {
_cartesianGrid[x, y].Active = false;
//_cartesianGrid[1, y].Active = true;
updateList.Add(new Point(1, y));
} else if( x < (arxlAmount - 1) ) {
_cartesianGrid[x, y].Active = false;
//_cartesianGrid[x + 1, y].Active = true;
updateList.Add(new Point(x + 1, y));
}
}
}
}
}
foreach( var pt in updateList ) {
_cartesianGrid[pt.X, pt.Y].Active = true;
}
}
In your timer try calling gridPictureBox.Invalidate() after you assign the image to the picturebox. This will force the picturebox to redraw itself.