I am implementing Yolov4-Tiny (onnx model found here) in Unity with the Windows ML APIs. I can load the model, and begin a session with no issue. I am using a VideoFrame (sized to 416x416) as the input and can access the two output Tensors. The problems arise when I begin to parse the output Tensors. With a confidence threshold of .5, I get between 700 and 1000 detections each frame, way way more than I expect. Also, the bboxes appear to be very small. The NMS and IOU functions below are nearly verbatim from here so I am not using anchors for the bounding boxes. I believe my issues is in the NMS and IOU functions, but I cannot locate the problem. My gut tells me I am manipulating the output Tensors incorrectly. Any ideas?
private List<DetectionResult> ParseResult(float[] boxes, float[] classes)
{
int c_values = 80;
int c_boxes = boxes.Length / 4;
int c_classNames = classes.Length / c_values;
float confidence_threshold = 0.5f;
List<DetectionResult> detections = new List<DetectionResult>();
for (int i_box = 0; i_box < c_classNames; i_box++)
{
float max_prob = 0.0f;
int label_index = -1;
for (int j_confidence = 0; j_confidence < c_values; j_confidence++)
{
int index = i_box * c_values + j_confidence;
if (Sigmoid(classes[index]) > max_prob)
{
max_prob = Sigmoid(classes[index]) ;
label_index = j_confidence;
}
}
if (max_prob > confidence_threshold)
{
//Debug.Log(_labels[label_index]);
List<float> bbox = new List<float>();
bbox.Add(boxes[(i_box * 4) + 0] * 416);
bbox.Add(boxes[(i_box * 4) + 1] * 416);
bbox.Add(boxes[(i_box * 4) + 2] * 416);
bbox.Add(boxes[(i_box * 4) + 3] * 416);
detections.Add(new DetectionResult()
{
label = _labels[label_index],
bbox = bbox,
prob = max_prob
});
}
}
private List<DetectionResult> NMS(IReadOnlyList<DetectionResult> detections,
float IOU_threshold = 0.45f,
float score_threshold = 0.3f)
{
List<DetectionResult> final_detections = new List<DetectionResult>();
for (int i = 0; i < detections.Count; i++)
{
int j = 0;
for (j = 0; j < final_detections.Count; j++)
{
if (ComputeIOU(final_detections[j], detections[i]) > IOU_threshold)
{
break;
}
}
if (j == final_detections.Count)
{
final_detections.Add(detections[i]);
}
}
return final_detections;
}
private float ComputeIOU(DetectionResult DRa, DetectionResult DRb)
{
float ay1 = DRa.bbox[0];
float ax1 = DRa.bbox[1];
float ay2 = DRa.bbox[2];
float ax2 = DRa.bbox[3];
float by1 = DRb.bbox[0];
float bx1 = DRb.bbox[1];
float by2 = DRb.bbox[2];
float bx2 = DRb.bbox[3];
// determine the coordinates of the intersection rectangle
float x_left = Math.Max(ax1, bx1);
float y_top = Math.Max(ay1, by1);
float x_right = Math.Min(ax2, bx2);
float y_bottom = Math.Min(ay2, by2);
if (x_right < x_left || y_bottom < y_top)
return 0;
float intersection_area = (x_right - x_left) * (y_bottom - y_top);
float bb1_area = (ax2 - ax1) * (ay2 - ay1);
float bb2_area = (bx2 - bx1) * (by2 - by1);
float iou = intersection_area / (bb1_area + bb2_area - intersection_area);
return iou;
}
Related
I'm working on writing a Mendelbrot renderer in C# to practice multithreading, but am having an issue where my calculation code maxes out at 2 iterations. I don't really understand why since the online references I've been looking at seem to calculate it the same way as me. No matter what coordinates of pixels I provide, it always returns 2 (aside from 0,0 and 0,1).
`
using System;
using System.Numerics;
using SkiaSharp;
namespace Mandelbrot
{
internal class Program
{
const int MaxIterations = 100;
static void Main(string[] args)
{
int size = 250;
int[,] grid = Run(size, 0, 0, 1);
// Console.WriteLine("Please specify a square size for the image.");
// try
// {
// Console.Write("Length: ");
// height = Int32.Parse(Console.ReadLine());
// }
// catch (FormatException e)
// {
// Console.WriteLine(e);
// throw;
// }
using (var surface = SKSurface.Create(width: size, height: size, SKColorType.Rgba8888, SKAlphaType.Premul))
{
SKCanvas canvas = surface.Canvas;
canvas.DrawColor(SKColors.Coral);
for (int i = 0; i < grid.GetLength(0); i++)
{
for (int j = 0; j < grid.GetLength(1); j++)
{
if (grid[i, j] >= MaxIterations)
canvas.DrawPoint(new SKPoint(i, j), SKColors.Black);
}
}
canvas.DrawPoint(new SKPoint(250, 250), SKColors.Chartreuse);
OutputImage(surface);
}
Console.WriteLine("Program successfully completed.");
}
public static int[,] Run(int size, int fromX, int fromY, int h)
{
int[] oulltput = new int[size * size];
int[,] output = new int[size, size];
for (int i = 0; i < output.GetLength(0); i += 1)
{
for (int j = 0; j < output.GetLength(1); j += 1)
{
float x = fromX + i * h;
float y = fromY + j * h;
output[i, j] = IterCount(x, y, MaxIterations);
}
}
return output;
}
public static int IterCount(float constX, float constY, int maxIterations)
{
const float maxMagnitude = 2f;
const float maxMagnitudeSquared = maxMagnitude * maxMagnitude;
int i = 0;
float x = 0.0f, y = 0.0f;
float xSquared = 0.0f, ySquared = 0.0f;
while (xSquared + ySquared <= maxMagnitudeSquared && i < maxIterations)
{
xSquared = x * x;
ySquared = y * y;
float xtmp = xSquared - ySquared + constX;
y = 2.0f * x * y + constY;
x = xtmp;
i++;
}
return i;
}
private static void OutputImage(SKSurface surface)
{
Console.WriteLine("Attempting to write .png to disk...");
using (var image = surface.Snapshot())
using (var data = image.Encode(SKEncodedImageFormat.Png, 80))
using (var stream = File.OpenWrite("out.png"))
{
// save the data to a stream
data.SaveTo(stream);
Console.WriteLine("Success!");
}
}
}
}
`
I tried to use breakpoints and writeline statements to debug but I can't figure out where my math is going wrong. I keep getting 2 for my iteration count.
The problem
I am trying to procedurally generate dungeon rooms with random X, Y sizes inside of a radius (r). However, even after I validate that the starting grid (origin of the "room") is not in the same position as other origins after running the separation function there are rooms still building inside of each other.
Solutions I have tried
I tried using math to calculate an optimal radius that will be able to fit the average of all the room sizes * amount of rooms. However, the separation should hypothetically work with any radius (though I want to keep them relatively close in order to keep hallways short).
Code
All my code is based on one tile. This means that all calculations are using one tile, and will remain one tile until the very end, then I scale them up.
private void GenerateRooms(int amount)
{
// init sizes
Vector2[] room_sizes = new Vector2[amount];
for (int i = 0; i < amount; i++)
{
room_sizes[i] = new Vector2(Random.Range(minimum_room_height, maximum_room_height), Random.Range(minimum_room_width, maximum_room_width));
}
float biggest_room = calculations.CalculateBiggest(room_sizes);
Vector2[] room_points = new Vector2[amount];
Vector2[] used_points = new Vector2[amount];
float radius = calculations.CalculateAverage(room_sizes) * amount;
for (int i = 0; i < amount; i++)
{
do {
Vector2 test_point = new Vector2(Random.Range(-radius, radius), Random.Range(-radius, radius));
foreach (Vector2 point in used_points) {
if (test_point == point) {
continue;
} else {
room_points[i] = test_point;
used_points[i] = test_point;
break;
}
}
} while (Vector2.Distance(Vector2.zero, room_points[i]) < radius);
}
for (int i = 0; i < amount; i++)
{
//Vector2 origin = room_points[i];
Vector3 position = calculations.computeSeperate(room_points, room_points[i], biggest_room);
//position = new Vector3(position.x + origin.x, position.y + origin.y, 0);
Vector3Int location = tile_map.WorldToCell(position);
tile_map.SetTile(location, tile);
calculations.scaleUpRooms(position, room_sizes[i].x, room_sizes[i].y, tile_map, tile);
}
}
The above is code for calling all the functions and validating the points. Here are the important functions (calculation functions):
public Vector2 computeSeperate(Vector2[] point_array, Vector2 target_point, float minimum_distance)
{
int neighbor_count = 0;
for (int i = 0; i < point_array.Length; i++)
{
if (point_array[i] != target_point)
{
if (Vector2.Distance(target_point, point_array[i]) < minimum_distance * 2)
{
target_point.x += point_array[i].x - target_point.x;
target_point.y += point_array[i].y - target_point.y;
neighbor_count++;
}
}
}
if (neighbor_count == 0)
{
return target_point;
} else
{
target_point.x /= neighbor_count;
target_point.y /= neighbor_count;
target_point.x *= -1;
target_point.y *= -1;
target_point.Normalize();
return target_point;
}
}
public void scaleUpRooms(Vector2 base_point, float scale_x, float scale_y, Tilemap tile_map, Tile tile) // ex: 5x5
{
List<Vector2> Calculate(Vector2 size)
{
List<Vector2> results = new List<Vector2>();
for (int i = 0; i < size.y; i++)
for (int o = 0; o < size.x; o++)
results.Add(new Vector2(o, i) + (new Vector2(size.x % 2 != 0 ? .5f : 1, size.y % 2 != 0 ? .5f : 1) - (size / 2)));
string st = "";
for (int i = 0; i < results.Count; i++)
st += "\n" + results[i].ToString();
return results;
}
Vector2 desired_scale = new Vector2(scale_x, scale_y);
List<Vector2> Offsets = Calculate(desired_scale);
for (int i = 0; i < Offsets.Count; i++)
{
Vector3 position = base_point + Offsets[i];
Vector3Int location = tile_map.WorldToCell(position);
tile_map.SetTile(location, tile);
}
}
I'm working on a Unity app that has some Microphone controls. At one point, I have to convert the Microphone input into Hertz (Hz) values and show them to the user. Now, I did some research and I made the following script for this purpose:
int amountSamples = 1024;
void Start ()
{
_fSample = AudioSettings.outputSampleRate;
}
void Update() {
if (focused && Initialized) {
if (Microphone.IsRecording(selectedDevice) && recording) {
spectrumData = GetSpectrumAnalysis();
if (spectrumCurve.keys.Length <= spectrumData.Length) {
float keyTimeValue = 0;
float currentHighestKeyTime = 0;
//create a curvefield if none exists
spectrumCurve = new AnimationCurve();
for (int t = 0; t < spectrumData.Length; t++) {
spectrumCurve.AddKey(1 / spectrumData.Length + t, spectrumData[t]);
spectrumCurve.MoveKey(1 / spectrumData.Length + t, new Keyframe(1 / spectrumData.Length + t, keyTimeValue = spectrumData[t])); //update keyframe value
if (keyTimeValue > currentHighestKeyTime) {
currentHighestKeyTime = keyTimeValue;
}
}
HighestKeyTimeValue = currentHighestKeyTime;
float freqN = HighestKeyTimeValue;
float f = freqN * (_fSample / 2) / amountSamples;
Debug.Log(f); //hz
}
}
}
audioSource.volume = 1;
}
And the GetSpectrumAnalysis()
public float[] GetSpectrumAnalysis ()
{
float[] dataSpectrum = new float[amountSamples];
audioSource.GetSpectrumData (dataSpectrum, 0, FFTWindow.BlackmanHarris);
for (int i = 0; i <= dataSpectrum.Length - 1; i++)
{
dataSpectrum[i] = Mathf.Abs (dataSpectrum[i] * sensitivity);
}
return dataSpectrum;
}
Now, with this code, the Hz value should be calculated in float f, it does work but the Hz values aren't too accurate, for example, I'm getting 400-500 Hz where I should get around 880 Hz. Similarly I'm getting 130 Hz instead of 220 Hz, etc.. So, I have 2 issues: I'm getting less Hz then I should and the Hz value is jumping too much and too fast so it's not consistent even if the sound playing is constant. Any idea how to improve this code? Where did I made a mistake?
EDIT
Check my answer for the solution.
Ok, nevermind, I found the solution, maybe this will help someone stumbling across this thread, change GetSpectrumAnalysis method to this:
public float test() {
float Threshold = 0.02f;
float[] dataSpectrum = new float[amountSamples];
audioSource.GetSpectrumData(dataSpectrum, 0, FFTWindow.BlackmanHarris); //Rectangular
float maxV = 0;
var maxN = 0;
for (int i = 0; i < amountSamples; i++) {
if (!(dataSpectrum[i] > maxV) || !(dataSpectrum[i] > Threshold)) {
continue;
}
maxV = dataSpectrum[i];
maxN = i; // maxN is the index of max
}
float freqN = maxN; // pass the index to a float variable
if (maxN > 0 && maxN < amountSamples - 1) { // interpolate index using neighbours
var dL = dataSpectrum[maxN - 1] / dataSpectrum[maxN];
var dR = dataSpectrum[maxN + 1] / dataSpectrum[maxN];
freqN += 0.5f * (dR * dR - dL * dL);
}
return freqN * (_fSample / 2) / amountSamples; // convert index to frequency
}
Then just call this in the update method like this:
Text.text = test().ToString("00");
For more info check out this thread: Unity answers
I'm coding an runtime terrain editor for unity and got stuck.
At first I wanted just to paint with textrue the terrain. I found this code and it worked fine:
SCRIPT: TerrainPainter
void Paint(Vector3 point)
{
mapX = (int)(((point.x - terrainPosition.x) / terrainData.size.x) * heightmapWidth);
mapY = (int)(((point.z - terrainPosition.z) / terrainData.size.z) * heigtmapHeight);
splatmapData[mapY, mapX, 0] = element[0, 0, 0] = 0;
splatmapData[mapY, mapX, 1] = element[0, 0, 1] = 1;
terrain.terrainData.SetAlphamaps(mapX, mapY, element);
}
But now I want to paint with different sizes/thickness. I have another script, named Terrainmodifier, which I use to raise and lower the terrain. There I have this lines for raising:
SCRIPT: Terrainmodifier
public void RaiseTerrain(Terrain terrain, Vector3 location, float effectIncrement)
{
int offset = areaOfEffectSize / 2;
//--1--
Vector3 tempCoord = (location - terrain.GetPosition());
Vector3 coord;
coord = new Vector3(
(tempCoord.x / GetTerrainSize().x),
(tempCoord.y / GetTerrainSize().y),
(tempCoord.z / GetTerrainSize().z)
);
Vector3 locationInTerrain = new Vector3(coord.x * terrainHeightMapWidth, 0, coord.z * terrainHeightMapHeight);
// End --1--
// --2--
int terX = (int)locationInTerrain.x - offset;
int terZ = (int)locationInTerrain.z - offset;
// End --2--
// --3--
float[,] heights = targetTerrainData.GetHeights(terX, terZ, areaOfEffectSize, areaOfEffectSize);
for (int xx = 0; xx < areaOfEffectSize; xx++)
{
for (int yy = 0; yy < areaOfEffectSize; yy++)
{
heights[xx, yy] += (effectIncrement * Time.smoothDeltaTime);
}
}
targetTerrainData.SetHeights(terX, terZ, heights);
}
So I thought I could use this as an aid and transfer it. So I took GetAlphamaps() instead of GetHeights() and added the variable areaOfEffectSize.
SCRIPT: TerrainPainter
void Paint(Vector3 point)
{
// --1--
mapX = (int)(((point.x - terrainPosition.x) / terrainData.size.x) * heightmapWidth);
mapY = (int)(((point.z - terrainPosition.z) / terrainData.size.z) * heigtmapHeight);
// End --1--
// --2--
int terX = (int)mapX - (areaOfEffectSize / 2);
int terY = (int)mapY - (areaOfEffectSize / 2);
// End --2--
// --3--
splatmapData = terrainData.GetAlphamaps(terX, terY, areaOfEffectSize, areaOfEffectSize);
for(int xx = 0; xx < areaOfEffectSize; xx++)
{
for (int yy = 0; yy < areaOfEffectSize; yy++)
{
splatmapData[yy, xx, 1] = element[0, 0, 1] = 1;
}
}
terrain.terrainData.SetAlphamaps(terX, terY, element);
}
Hope sb can help me find my mistake. How can I change the size of my "brush"?
EDIT: I wrote comments into the code to see the transfered/related lines.
Oh guys, I did a stupid mistake. Solved this problem by passing the splatmapData into SetAlphamaps -.-
So the solution is:
[..]
for (int xx = 0; xx < areaOfEffectSize; xx++)
{
for (int yy = 0; yy < areaOfEffectSize; yy++)
{
splatmapData[yy, xx, 0] = 0;
splatmapData[yy, xx, 1] = 1;
}
}
terrain.terrainData.SetAlphamaps(terX, terY, splatmapData);
I am trying to implement a fluid surface simulation using the paper Fast Hydraulic Erosion Simulation and Visualization on GPU as a template to simulate water. However, I get artifacts like this:
The code for my update function is below, I have followed another post on this subject, but the change did not help.
private void updateHeight (float dt, float dx){
float dhL, dhR, dhF, dhB, DV;
float totalFlux;
float updateConstant;
float fluxConstant = dt*GRAVITY/(2*VISCOUSCOEFFICEINT*dx);
//Update Flux
for (int y=1; y<=N-1; y++){
for(int x=1;x<=N-1;x++){
dhL = this.height[x][y] - this.height[x-1][y];
dhR = this.height[x][y] - this.height[x+1][y];
dhF = this.height[x][y] - this.height[x][y+1];
dhB = this.height[x][y] - this.height[x][y-1];
if (Mathf.Abs(dhL) < 0.0001f){
dhL=0.0f;
}
if (Mathf.Abs(dhR) < 0.0001f){
dhR=0;
}
if (Mathf.Abs(dhF) < 0.0001f){
dhF=0;
}
if (Mathf.Abs(dhB) < 0.0001f){
dhB=0;
}
this.tempFluxArray[x][y].fluxL = Mathf.Max(0.0f, this.fluxArray[x][y].fluxL + fluxConstant*dhL);
this.tempFluxArray[x][y].fluxR = Mathf.Max(0.0f, this.fluxArray[x][y].fluxR + fluxConstant*dhR);
this.tempFluxArray[x][y].fluxF = Mathf.Max(0.0f, this.fluxArray[x][y].fluxF + fluxConstant*dhF);
this.tempFluxArray[x][y].fluxB = Mathf.Max(0.0f, this.fluxArray[x][y].fluxB + fluxConstant*dhB);
totalFlux = this.tempFluxArray[x][y].fluxL + this.tempFluxArray[x][y].fluxR + this.tempFluxArray[x][y].fluxF + this.tempFluxArray[x][y].fluxB;
if(totalFlux > 0){
updateConstant = Mathf.Min(1.0f, this.height[x][y]* dx*dx/(totalFlux * dt));
this.tempFluxArray[x][y].fluxL = updateConstant * this.tempFluxArray[x][y].fluxL;
this.tempFluxArray[x][y].fluxR = updateConstant * this.tempFluxArray[x][y].fluxR;
this.tempFluxArray[x][y].fluxF = updateConstant * this.tempFluxArray[x][y].fluxF;
this.tempFluxArray[x][y].fluxB = updateConstant * this.tempFluxArray[x][y].fluxB;
}
}
}
swap();
//Height Calculation
for (int y=1; y<=N-1; y++){
for(int x=1;x<=N-1;x++){
DV = dt*(this.fluxArray[x-1][y].fluxR + this.fluxArray[x][y-1].fluxF + this.fluxArray[x+1][y].fluxL + this.fluxArray[x][y+1].fluxB - this.fluxArray[x][y].fluxL - this.fluxArray[x][y].fluxR - this.fluxArray[x][y].fluxF - this.fluxArray[x][y].fluxB);
this.height[x][y] = this.height[x][y] + DV/(dx*dx);
if(this.height[x][y] < 1){
// Debug.Log(x);
// Debug.Log(y);
}
}
}
}
Could it be due to using this rather than ref? The way I interact with the water surface written below.
private void waterdrop(int x, int y){
float sqrD = 0.8f*0.8f;
for (int j = 1; j < N; j++) {
for (int i = 1; i < N; i++) {
float sqrDToVert = (float)0.2f*(i - x)*(i-x) + 0.2f*(j - y)*(j-y);
if (sqrDToVert <= sqrD){
float distanceCompensator = 1 - (sqrDToVert/sqrD);
this.fluxArray[i][j].fluxL = this.fluxArray[i][j].fluxL + (0.02f * distanceCompensator);
this.fluxArray[i][j].fluxR = this.fluxArray[i][j].fluxR + (0.02f * distanceCompensator);
this.fluxArray[i][j].fluxF = this.fluxArray[i][j].fluxF + (0.02f * distanceCompensator);
this.fluxArray[i][j].fluxB = this.fluxArray[i][j].fluxB + (0.02f * distanceCompensator);
//this.height[i][j] = this.height[i][j] - 1.0f * distanceCompensator;
Debug.Log(this.height[i][j]);
}
//Debug.Log("x = "+i+"\n y = "+j+" height is "+this.height[i][j]);
}
}
}
Namely just changing the flux at some point at all direction.
Some solutions I have tried was to change the height but that didn't work or clamp on small changes only but it only built up. The boundary conditions are zero flux on all boarders and the height would be the same as the closest point.