We Keep Coding

Applying image over detected face in webcam using OpenCvSharp

I'm trying to apply a simple, rectangular image over faces that are detected in a webcam utilizing of OpenCVSharp. I did manage to make the detecting part by applying a rectangle around the faces, but I'm not sure how to be able to apply an image of the size (and position) of the region of interest.
I created a Mat for the image, but when I copy that image to the display frame, it takes over the entire dimension of the frame instead of only the face region.
How can I apply the desired effect of an image over the marked face region? Do I need to blend two Mats togheter? If so, how?
Code for the face detection
using UnityEngine;
using OpenCvSharp;
public class FaceDetection : MonoBehaviour
{
CascadeClassifier cascade;
public bool initialized = false;
public int totalFaces = 0;
public Mat maskImage;
private void OnEnable()
{
WebcamManager.onFrameProcessed += ProcessFrame;
}
private void OnDisable()
{
WebcamManager.onFrameProcessed -= ProcessFrame;
}
public void Start()
{
var dataPath = Application.dataPath;
//For some reason, Cv2.ImRead always returned an empty Mat, so I created my own class for the conversion, it will be below this code.
maskImage = MyConverter.ConvertToMat(Application.dataPath + #"/Sprites/facemask.png"); //This image is 420x420
Debug.Log(maskImage); //this returns: Mat [ 420*420*CV_8UC1, IsContinuous=True, IsSubmatrix=False, Ptr=0x23eae88bf50, Data=0x23ee94f0040 ]
cascade = new CascadeClassifier(Application.dataPath + #"/haarcascade_frontalface_default.xml");
initialized = true;
}
//This is called in a delegate of a Webcam script, all it does is update the webcam frame and then call this method.
public void ProcessFrame(WebCamTexture texture)
{
if (texture == null || !initialized) return;
Mat frame = OpenCvSharp.Unity.TextureToMat(texture);
var faces = cascade.DetectMultiScale(frame,1.2,1,HaarDetectionType.ScaleImage);
if (faces.Length >= 1)
{
for (int i = 0; i < faces.Length; i++)
{
DisplayFace(frame, faces[i]);
}
}
totalFaces = faces.Length;
}
public void DisplayFace(Mat frame, OpenCvSharp.Rect face)
{
Debug.Log("Face width and height: " + face.Height.ToString() + " x " + face.Width.ToString());
//Here I try to resize the image that is to be applied over the faces, but it ends up taking the whole frame...
Cv2.Resize(maskImage, maskImage, new OpenCvSharp.Size(face.Height, face.Width));
maskImage.CopyTo(frame);
//Draws a blue square around all detected faces.
frame.Rectangle(face, new Scalar(250,0,0), 2);
Texture newTexture = OpenCvSharp.Unity.MatToTexture(frame);
WebcamManager.instance.rend.GetComponent<Renderer>().material.mainTexture = newTexture;
}
}
Converter Class
using UnityEngine;
using OpenCvSharp;
using System.IO;
public static class MyConverter
{
//https://stackoverflow.com/questions/57497491/opencvsharp-imread-function-cant-load-image-in-unity-on-macos
public static Mat ConvertToMat(string filePath)
{
Mat matResult = null;
byte[] fileData = File.ReadAllBytes(filePath);
var tex = new Texture2D(420,420);
tex.LoadImage(fileData);
matResult = OpenCvSharp.Unity.TextureToMat(tex);
Cv2.CvtColor(matResult, matResult, ColorConversionCodes.BGR2GRAY);
return matResult;
}
}
Code for the Webcam Manager
using UnityEngine;
public class WebcamManager : MonoBehaviour
{
public static WebcamManager instance;
public delegate void ProcessFrame(WebCamTexture _wbt);
public static event ProcessFrame onFrameProcessed;
[Header("Settings")]
public int startingIndex = 0; //The index of the camera that will be started by the manager.
[Header("Components")]
public Renderer rend;
private WebCamTexture wbt;
[Header("Status")]
public bool isPlaying = false;
private void Awake()
{
if (!instance) instance = this;
}
private void Start()
{
StartWebcam(0);
}
public void StartWebcam(int index)
{
WebCamDevice[] devices = WebCamTexture.devices;
if (devices.Length <= 0)
{
Debug.LogError("[WebcamManager] There are no camera devices attached to the system!");
return;
}
if (index < devices.Length)
{
Debug.Log("[WebcamManager] Playing camera device: " + devices[index].name);
wbt = new WebCamTexture(devices[index].name);
wbt.Play();
isPlaying = true;
}
}
public void StopWebcam()
{
if (wbt) wbt.Stop();
isPlaying = false;
}
public void UpdateWebcamTexture()
{
if (!isPlaying) return;
if (rend)
{
rend.material.mainTexture = wbt;
}
if (onFrameProcessed != null && wbt) onFrameProcessed(wbt);
}
private void Update()
{
UpdateWebcamTexture();
}
}

How do you display a radius around an object using sqrMagnitude

Im trying to get a radius to display around an object I want to be able to set one variable and set an object to match the radius of sqrMagnitude.
public class ObjectFinder : MonoBehaviour
{
public GameObject player;
private GameObject objectToFind;
public GameObject objectRadius;
public float radius;
// Start is called before the first frame update
void Start()
{
}
// Update is called once per frame
void Update()
{
objectRadius.transform.localScale = new Vector3(radius,radius,0);
obj1 = GameObject.FindWithTag("ObjectFind");
if ((objectToFind.transform.position - player.transform.position).sqrMagnitude < radius * 1.75)
{
Debug.Log("Object Found");
}
else
{
Debug.Log("Object out of range");
}
}
}

Custom window mimicking SceneView

I am building a custom window and I am trying to reuse Unity's Scene view to be able to draw directly from this specific window.
I manage to reproduce the correct window by extend UnityEditor.SceneView and here's what I have:
And here's the code:
[EditorWindowTitle(title = "Shape Editor", useTypeNameAsIconName = false)]
public class StrokeEditor : SceneView
{
[MenuItem("Recognizer/Shape Editor")]
public static void Init()
{
var w = GetWindow<StrokeEditor>();
w.in2DMode = true;
EditorSceneManager.NewScene(NewSceneSetup.EmptyScene, NewSceneMode.Single);
}
protected override void OnGUI()
{
using (new GUILayout.HorizontalScope())
{
GUILayout.Button("Add Stroke");
GUILayout.Button("Edit Stroke");
GUILayout.Button("Delete Stroke");
}
base.OnGUI();
}
}
With this, I might be almost done.
Is this the right way to procede ?
I feel that something is wrong because whenever I use EditorSceneManager.NewScene(NewSceneSetup.EmptyScene, NewSceneMode.Single);, it creates also a new scene to the main scene view. (I want the main scene view to stay unchanged)
I should also be able to see the tools from the scene view like:
Is there any better way to achieve what I want ?
EDIT 1:
The final usage of all of this is to be able to draw 2D shapes within the window by clicking and dragging the mouse like gestures with mobile phones. From that, I'll be able to get some of the position to feed one of my algorithm...

You can use the new GraphView. This gives you some of the things you are looking for for "free", mainly to zoom and pan the view. Since ShaderGraph uses this, it should be easier to construct nodes, select them and move them around, if that is something you want to.
Here is a toy example of a custom editor window that allows you to edit list of points in a scriptable object:
Shape.cs - simple scriptable object with a list of points.
[CreateAssetMenu(menuName = "Test/ShapeObject")]
public class Shape : ScriptableObject
{
public List<Vector2> PointList = new List<Vector2>();
}
ShapeEditorWindow.cs - editor window with a toolbar and a graphview that opens scriptable objects of type Shape.
using UnityEngine;
using UnityEditor;
using UnityEditor.UIElements;
public class ShapeEditorWindow : EditorWindow
{
private ShapeEditorGraphView _shapeEditorGraphView;
private Shape _shape;
[UnityEditor.Callbacks.OnOpenAsset(1)]
private static bool Callback(int instanceID, int line)
{
var shape = EditorUtility.InstanceIDToObject(instanceID) as Shape;
if (shape != null)
{
OpenWindow(shape);
return true;
}
return false; // we did not handle the open
}
private static void OpenWindow(Shape shape)
{
var window = GetWindow<ShapeEditorWindow>();
window.titleContent = new GUIContent("Shape Editor");
window._shape = shape;
window.rootVisualElement.Clear();
window.CreateGraphView();
window.CreateToolbar();
}
private void CreateToolbar()
{
var toolbar = new Toolbar();
var clearBtn = new ToolbarButton(()=>_shape.PointList.Clear()); ;
clearBtn.text = "Clear";
var undoBtn = new ToolbarButton(() =>_shape.PointList.RemoveAt(_shape.PointList.Count-1));
undoBtn.text = "Undo";
toolbar.Add(clearBtn);
toolbar.Add(new ToolbarSpacer());
toolbar.Add(undoBtn);
rootVisualElement.Add(toolbar);
}
private void CreateGraphView()
{
_shapeEditorGraphView = new ShapeEditorGraphView(_shape);
_shapeEditorGraphView.name = "Shape Editor Graph";
rootVisualElement.Add(_shapeEditorGraphView);
}
}
ShapeEditorGraphView.cs - graphview with zoom, grid, pan (with ContentDragger) and shape editor.
using UnityEditor.Experimental.GraphView;
using UnityEngine;
using UnityEngine.UIElements;
public class ShapeEditorGraphView : GraphView
{
const float _pixelsPerUnit = 100f;
const bool _invertYPosition = true;
public ShapeEditorGraphView(Shape shape){
styleSheets.Add(Resources.Load<StyleSheet>("ShapeEditorGraph"));
this.StretchToParentSize();
SetupZoom(ContentZoomer.DefaultMinScale, ContentZoomer.DefaultMaxScale);
Add(new GridBackground());
//pan with Alt-LeftMouseButton drag/ MidleMouseButton drag
this.AddManipulator(new ContentDragger());
//other things that might interest you
//this.AddManipulator(new SelectionDragger());
//this.AddManipulator(new RectangleSelector());
//this.AddManipulator(new ClickSelector());
this.AddManipulator(new ShapeManipulator(shape));
contentViewContainer.BringToFront();
contentViewContainer.Add(new Label { name = "origin", text = "(0,0)" });
//set the origin to the center of the window
this.schedule.Execute(() =>
{
contentViewContainer.transform.position = parent.worldBound.size / 2f;
});
}
public Vector2 WorldtoScreenSpace(Vector2 pos)
{
var position = pos * _pixelsPerUnit - contentViewContainer.layout.position;
if (_invertYPosition) position.y = -position.y;
return contentViewContainer.transform.matrix.MultiplyPoint3x4(position);
}
public Vector2 ScreenToWorldSpace(Vector2 pos)
{
Vector2 position = contentViewContainer.transform.matrix.inverse.MultiplyPoint3x4(pos);
if (_invertYPosition) position.y = -position.y;
return (position + contentViewContainer.layout.position) / _pixelsPerUnit;
}
}
Unfortunately the grid background and the grid lines are the same color, so in order to see the grid lines we have to write a style sheet and set the GridBackground properties. This file has to be in Editor/Resources, and gets loaded with styleSheets.Add(Resources.Load<StyleSheet>("ShapeEditorGraph"));
Editor/Resources/ShapeEditorGraph.uss
GridBackground {
--grid-background-color: rgba(32,32,32,1);
--line-color: rgba(255,255,255,.1);
--thick-line-color: rgba(255,255,255,.3);
--spacing: 100;
}
ShapeManipulator.cs - draws and edits the shape. This is similar to RectangleSelector.
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
using UnityEngine.UIElements;
public class ShapeManipulator : MouseManipulator
{
private Shape _shape;
private ShapeDraw _shapeDraw;
public ShapeManipulator(Shape shape)
{
activators.Add(new ManipulatorActivationFilter { button = MouseButton.LeftMouse });
_shape = shape;
_shapeDraw = new ShapeDraw { points = shape.PointList };
}
protected override void RegisterCallbacksOnTarget()
{
target.Add(_shapeDraw);
target.Add(new Label { name = "mousePosition", text = "(0,0)" });
target.RegisterCallback<MouseDownEvent>(MouseDown);
target.RegisterCallback<MouseMoveEvent>(MouseMove);
target.RegisterCallback<MouseCaptureOutEvent>(MouseOut);
target.RegisterCallback<MouseUpEvent>(MouseUp);
}
protected override void UnregisterCallbacksFromTarget()
{
target.UnregisterCallback<MouseDownEvent>(MouseDown);
target.UnregisterCallback<MouseUpEvent>(MouseUp);
target.UnregisterCallback<MouseMoveEvent>(MouseMove);
target.UnregisterCallback<MouseCaptureOutEvent>(MouseOut);
}
private void MouseOut(MouseCaptureOutEvent evt) => _shapeDraw.drawSegment = false;
private void MouseMove(MouseMoveEvent evt)
{
var t = target as ShapeEditorGraphView;
var mouseLabel = target.Q("mousePosition") as Label;
mouseLabel.transform.position = evt.localMousePosition + Vector2.up * 20;
mouseLabel.text = t.ScreenToWorldSpace(evt.localMousePosition).ToString();
//if left mouse is pressed
if ((evt.pressedButtons & 1) != 1) return;
_shapeDraw.end = t.ScreenToWorldSpace(evt.localMousePosition);
_shapeDraw.MarkDirtyRepaint();
}
private void MouseUp(MouseUpEvent evt)
{
if (!CanStopManipulation(evt)) return;
target.ReleaseMouse();
if (!_shapeDraw.drawSegment) return;
if (_shape.PointList.Count == 0) _shape.PointList.Add(_shapeDraw.start);
var t = target as ShapeEditorGraphView;
_shape.PointList.Add(t.ScreenToWorldSpace(evt.localMousePosition));
_shapeDraw.drawSegment = false;
_shapeDraw.MarkDirtyRepaint();
}
private void MouseDown(MouseDownEvent evt)
{
if (!CanStartManipulation(evt)) return;
target.CaptureMouse();
_shapeDraw.drawSegment = true;
var t = target as ShapeEditorGraphView;
if (_shape.PointList.Count != 0) _shapeDraw.start = _shape.PointList.Last();
else _shapeDraw.start = t.ScreenToWorldSpace(evt.localMousePosition);
_shapeDraw.end = t.ScreenToWorldSpace(evt.localMousePosition);
_shapeDraw.MarkDirtyRepaint();
}
private class ShapeDraw : ImmediateModeElement
{
public List<Vector2> points { get; set; } = new List<Vector2>();
public Vector2 start { get; set; }
public Vector2 end { get; set; }
public bool drawSegment { get; set; }
protected override void ImmediateRepaint()
{
var lineColor = new Color(1.0f, 0.6f, 0.0f, 1.0f);
var t = parent as ShapeEditorGraphView;
//Draw shape
for (int i = 0; i < points.Count - 1; i++)
{
var p1 = t.WorldtoScreenSpace(points[i]);
var p2 = t.WorldtoScreenSpace(points[i + 1]);
GL.Begin(GL.LINES);
GL.Color(lineColor);
GL.Vertex(p1);
GL.Vertex(p2);
GL.End();
}
if (!drawSegment) return;
//Draw current segment
GL.Begin(GL.LINES);
GL.Color(lineColor);
GL.Vertex(t.WorldtoScreenSpace(start));
GL.Vertex(t.WorldtoScreenSpace(end));
GL.End();
}
}
}
The is just example code. The goal was to have something working and drawing to the screen.

I have tackled something similar to this in the past. When I wanted to extend the SceneView, I've used Gizmos and drawing callbacks to add my own controls to the scene view, but I suspect you might want more freedom than just this.
The other thing I did was to create an "editor preview scene", add a camera to it and made the camera render into my custom EditorWindow. It's a lot of work, but once I did it I was completely free to customize the editor experience.
It's probably quite dangerous inheriting from Unity's SceneView, as I expect that it will change so often that you may struggle to get your stuff working on multiple versions. You might also find yourself breaking stuff when Unity's code doesn't expect anyone to be inheriting from SceneView.

Trying to find all of the objects in my scene by a custom tag script I have created, however i continue getting a null reference exception

The Tag script has been working for me so far as an alternate to the unity tags up to this point, allowing me to assign multiple tags to an object at once. Now I want to create a method that will get all of the objects in the scene, filter them by the tag, and then return it as an array. The null reference exception refers to line 41 of the Tag.cs script. How do I fix this?
Tags.cs file
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class Tags : MonoBehaviour
{
public string[] startTags;
private string[] tags;
private void Start()
{
tags = startTags;
}
public bool FindTag(string search)
{
bool results = false;
for (int i = 0; i < tags.Length; i++)
{
if(search == tags[i])
{
results = true;
break;
}
}
return results;
}
//Find objects by custom script tags
//HERE IS WHERE THE METHOD IS CREATED
public static GameObject[] ObjectsByTag(string search)
{
//Get all objects in scene
GameObject[] allObjects = FindObjectsOfType<GameObject>();
GameObject[] storedObjects = new GameObject[allObjects.Length];
GameObject[] finalObjects;
//Filter
int count = 0;
for (int i = 0; i < allObjects.Length; i++)
{
if (allObjects[i].GetComponent<Tags>().FindTag(search)) //line 41
{
storedObjects[count] = allObjects[i];
count++;
}
}
//Assign final length
finalObjects = new GameObject[count];
//Reassign to final array
for (int i = 0; i < count; i++)
{
finalObjects[i] = storedObjects[i];
}
return finalObjects;
}
}
GameController.cs file (How it is being used
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class GameController : MonoBehaviour
{
//SCREEN START
//Get Screen Size
private float sHeight;
private float sWidth;
//Intended Screen Size
readonly float iH = 695f;
readonly float iW = 1540f;
//Convert
private float cH;
private float cW;
public float ConvertedHeight => cH;
public float ConvertedWidth => cW;
//SCREEN END
//MOUSE CAM START
//mousePostion
private float mX;
private float mZ;
public float MouseX => mX;
public float MouseZ => mZ;
//MOUSE CAM END
//EnemySpeedModifier
private float esm;
public float ESM
{
get { return esm; }
set { esm = value; }
}
//GameOver
private bool gameOver = false;
public bool GameOver
{
get { return gameOver; }
set { gameOver = value; }
}
//game speed
public float speed;
/*
//projectile list
private GameObject[] projectiles;
public GameObject[] Projectiles()
{
return projectiles;
}
public void Projectiles(GameObject value)
{
GameObject[] tempArray = projectiles;
tempArray[projectiles.Length] = value;
projectiles = tempArray;
Debug.Log("Projectile Count: " + projectiles.Length);
}
*/
//HERE IS WHERE IT IS USED
public GameObject[] ProjectilesInScene
{
get
{
return Tags.ObjectsByTag("projectile");
}
}
// Start is called before the first frame update
void Start()
{
//CONVERT SCREEN SIZES START
sHeight = Screen.height;
sWidth = Screen.width;
cH = iH / sHeight;
cW = iW / sWidth;
//CONVERT SCREEN SIZES END
}
// Update is called once per frame
void Update()
{
if (gameOver)
{
speed /= 1 + 0.5f * Time.deltaTime;
}
//Update mose position
mX = Input.mousePosition.x;
mZ = Input.mousePosition.y;
}
}

It seems that not all of your GameObject objects have the Tags component. Per the GameObject.GetComponent documentation
Returns the component of Type type if the game object has one attached, null if it doesn't.
If you know that some objects won't have the Tags component, your line 41 can use a simple null conditional operator:
if (allObjects[i].GetComponent<Tags>()?.FindTag(search) == true)
{
...
}
Note the ? after GetComponent<Tags>().

How can i use ui button and ui toggle in a script to create/destroy objects on another script?

In the Hierarchy i have a Canvas and under it as childs i have a ui button and ui toggle. On the button i attached a script with a onclick event.
The idea is when i click the button depending on the toggle state false/true decide if to just create more gameobjects or destroy first all the old gameobjects and create new.
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
public class GenerateObjectsButton : MonoBehaviour
{
private InstantiateObjects instantiateobjects;
private bool toggleOnOf;
public Toggle toggle;
private void Start()
{
instantiateobjects = new InstantiateObjects();
toggle.onValueChanged.AddListener((value) =>
{
MyListener(value);
});
}
public void MyListener(bool value)
{
if (value)
{
//do the stuff when the toggle is on
toggleOnOf = true;
}
else
{
//do the stuff when the toggle is off
toggleOnOf = false;
}
}
public void OnButton()
{
if (toggleOnOf == false)
{
instantiateobjects.generateObjectOnTerrain();
}
else
{
instantiateobjects.DestroyObjects();
instantiateobjects.generateObjectOnTerrain();
}
}
}
The InstantiateObjects script:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
//[ExecuteInEditMode]
public class InstantiateObjects : MonoBehaviour
{
public GameObject prefab;
public Terrain terrain;
public float yOffset = 0.5f;
public int objectsToInstantiate;
public bool parent = true;
public bool randomScale = false;
public float setRandScaleXMin, setRandScaleXMax;
public float setTandScaleYMin, setTandScaleYMax;
public float setTandScaleZMin, setRandScaleZMax;
public bool generateNew;
private float terrainWidth;
private float terrainLength;
private float xTerrainPos;
private float zTerrainPos;
private int numberOfObjectsToCreate;
private GameObject objInstance;
private GameObject[] createdObjects;
private string objname;
public void Start()
{
//Get terrain size
terrainWidth = terrain.terrainData.size.x;
terrainLength = terrain.terrainData.size.z;
//Get terrain position
xTerrainPos = terrain.transform.position.x;
zTerrainPos = terrain.transform.position.z;
numberOfObjectsToCreate = objectsToInstantiate;
objname = prefab.name;
MyCustomEditor.TagsAndLayers.AddTag(objname);
generateNew = false;
generateObjectOnTerrain();
}
public void Update()
{
}
public void DestroyObjects()
{
if (createdObjects != null && createdObjects.Length > 0)
{
for (int i = 0; i < createdObjects.Length; i++)
{
DestroyImmediate(createdObjects[i]);
}
createdObjects = new GameObject[0];
}
}
public void generateObjectOnTerrain()
{
for (int i = 0; i < objectsToInstantiate; i++)
{
//Generate random x,z,y position on the terrain
float randX = UnityEngine.Random.Range(xTerrainPos, xTerrainPos + terrainWidth);
float randZ = UnityEngine.Random.Range(zTerrainPos, zTerrainPos + terrainLength);
float yVal = Terrain.activeTerrain.SampleHeight(new Vector3(randX, 0, randZ));
//Generate random x,y,z scale on the terrain
float randScaleX = Random.Range(setRandScaleXMin, setRandScaleXMax);
float randScaleY = Random.Range(setTandScaleYMin, setTandScaleYMax);
float randScaleZ = Random.Range(setTandScaleYMax, setRandScaleZMax);
//Apply Offset if needed
yVal = yVal + yOffset;
//Generate the Prefab on the generated position
objInstance = Instantiate(prefab, new Vector3(randX, yVal, randZ), Quaternion.identity);
if (randomScale == true)
objInstance.transform.localScale = new Vector3(randScaleX, randScaleY, randScaleZ);
if (parent)
objInstance.transform.parent = this.transform;
objInstance.tag = objname;
}
createdObjects = GameObject.FindGameObjectsWithTag(objname);
}
}
The problems i'm facing now are:
In the InstantiateObjects when i click the button i'm getting null exception on the line:
createdObjects = GameObject.FindGameObjectsWithTag(objname);
createdObjects is null. Except the first time i'm running the game then it's not null but then each time when i click the button it's null.
Second problem is that i need to change the toggle state to false then click the button true click the button false click the button and only then it's getting the DestroyObjects method.

It might not be updating the array as the reference are the same. Reset the array to null at the start of each iteration and see if that works.