Applying image over detected face in webcam using OpenCvSharp - c#

I'm trying to apply a simple, rectangular image over faces that are detected in a webcam utilizing of OpenCVSharp. I did manage to make the detecting part by applying a rectangle around the faces, but I'm not sure how to be able to apply an image of the size (and position) of the region of interest.
I created a Mat for the image, but when I copy that image to the display frame, it takes over the entire dimension of the frame instead of only the face region.
How can I apply the desired effect of an image over the marked face region? Do I need to blend two Mats togheter? If so, how?
Code for the face detection
using UnityEngine;
using OpenCvSharp;
public class FaceDetection : MonoBehaviour
{
CascadeClassifier cascade;
public bool initialized = false;
public int totalFaces = 0;
public Mat maskImage;
private void OnEnable()
{
WebcamManager.onFrameProcessed += ProcessFrame;
}
private void OnDisable()
{
WebcamManager.onFrameProcessed -= ProcessFrame;
}
public void Start()
{
var dataPath = Application.dataPath;
//For some reason, Cv2.ImRead always returned an empty Mat, so I created my own class for the conversion, it will be below this code.
maskImage = MyConverter.ConvertToMat(Application.dataPath + #"/Sprites/facemask.png"); //This image is 420x420
Debug.Log(maskImage); //this returns: Mat [ 420*420*CV_8UC1, IsContinuous=True, IsSubmatrix=False, Ptr=0x23eae88bf50, Data=0x23ee94f0040 ]
cascade = new CascadeClassifier(Application.dataPath + #"/haarcascade_frontalface_default.xml");
initialized = true;
}
//This is called in a delegate of a Webcam script, all it does is update the webcam frame and then call this method.
public void ProcessFrame(WebCamTexture texture)
{
if (texture == null || !initialized) return;
Mat frame = OpenCvSharp.Unity.TextureToMat(texture);
var faces = cascade.DetectMultiScale(frame,1.2,1,HaarDetectionType.ScaleImage);
if (faces.Length >= 1)
{
for (int i = 0; i < faces.Length; i++)
{
DisplayFace(frame, faces[i]);
}
}
totalFaces = faces.Length;
}
public void DisplayFace(Mat frame, OpenCvSharp.Rect face)
{
Debug.Log("Face width and height: " + face.Height.ToString() + " x " + face.Width.ToString());
//Here I try to resize the image that is to be applied over the faces, but it ends up taking the whole frame...
Cv2.Resize(maskImage, maskImage, new OpenCvSharp.Size(face.Height, face.Width));
maskImage.CopyTo(frame);
//Draws a blue square around all detected faces.
frame.Rectangle(face, new Scalar(250,0,0), 2);
Texture newTexture = OpenCvSharp.Unity.MatToTexture(frame);
WebcamManager.instance.rend.GetComponent<Renderer>().material.mainTexture = newTexture;
}
}
Converter Class
using UnityEngine;
using OpenCvSharp;
using System.IO;
public static class MyConverter
{
//https://stackoverflow.com/questions/57497491/opencvsharp-imread-function-cant-load-image-in-unity-on-macos
public static Mat ConvertToMat(string filePath)
{
Mat matResult = null;
byte[] fileData = File.ReadAllBytes(filePath);
var tex = new Texture2D(420,420);
tex.LoadImage(fileData);
matResult = OpenCvSharp.Unity.TextureToMat(tex);
Cv2.CvtColor(matResult, matResult, ColorConversionCodes.BGR2GRAY);
return matResult;
}
}
Code for the Webcam Manager
using UnityEngine;
public class WebcamManager : MonoBehaviour
{
public static WebcamManager instance;
public delegate void ProcessFrame(WebCamTexture _wbt);
public static event ProcessFrame onFrameProcessed;
[Header("Settings")]
public int startingIndex = 0; //The index of the camera that will be started by the manager.
[Header("Components")]
public Renderer rend;
private WebCamTexture wbt;
[Header("Status")]
public bool isPlaying = false;
private void Awake()
{
if (!instance) instance = this;
}
private void Start()
{
StartWebcam(0);
}
public void StartWebcam(int index)
{
WebCamDevice[] devices = WebCamTexture.devices;
if (devices.Length <= 0)
{
Debug.LogError("[WebcamManager] There are no camera devices attached to the system!");
return;
}
if (index < devices.Length)
{
Debug.Log("[WebcamManager] Playing camera device: " + devices[index].name);
wbt = new WebCamTexture(devices[index].name);
wbt.Play();
isPlaying = true;
}
}
public void StopWebcam()
{
if (wbt) wbt.Stop();
isPlaying = false;
}
public void UpdateWebcamTexture()
{
if (!isPlaying) return;
if (rend)
{
rend.material.mainTexture = wbt;
}
if (onFrameProcessed != null && wbt) onFrameProcessed(wbt);
}
private void Update()
{
UpdateWebcamTexture();
}
}

Related

Cant seem to get the player position with Vector3 with unity

How would i get the player position so i can drop the item about 5 units infrount of the player so they do not imeadently pick up the item after they drop it and how do i get the Vector3 position of the player as well tried in many ways and i still cant get it to work i am trying to do it here:
public void DropItem()
{
if(slotsItem)
{
slotsItem.transform.parent = null;
slotsItem.gameObject.SetActive(true);
slotsItem.transform.position = Vector3.lastpos;
}
then here is the full code as well
using System.Collections;
using System.Collections.Generic;
using UnityEngine.UI;
using UnityEngine;
public class Slot : MonoBehaviour
{
public Item slotsItem;
public Transform player;
Sprite defaultSprite;
Text amountText;
public void CustomStart()
{
defaultSprite = GetComponent<Image>().sprite;
amountText = transform.GetChild(0).GetComponent<Text>();
this.player = GameObject.FindWithTag("Player").transform;
Vector3 lastpos = player.position;
}
public void DropItem()
{
if(slotsItem)
{
slotsItem.transform.parent = null;
slotsItem.gameObject.SetActive(true);
slotsItem.transform.position = Vector3.lastpos;
}
}
public void CheckForItem()
{
if(transform.childCount > 1)
{
slotsItem = transform.GetChild(1).GetComponent<Item>();
GetComponent<Image>().sprite = slotsItem.itemSprite;
if(slotsItem.amountInStack > 1)
amountText.text = slotsItem.amountInStack.ToString();
}
else
{
slotsItem = null;
GetComponent<Image>().sprite = defaultSprite;
amountText.text = "";
}
}
}
What is Vector3.lastpos supposed to be?
I think you rather want to store a field in the class
private Vector3 lastpos;
assign this field in
public void CustomStart()
{
...
lastpos = player.position;
}
and then later use it like
public void DropItem()
{
...
slotsItem.transform.position = lastpos;
...
}
though the question is why not simply use the current position
slotsItem.transform.position = player.position;

Trying to find all of the objects in my scene by a custom tag script I have created, however i continue getting a null reference exception

The Tag script has been working for me so far as an alternate to the unity tags up to this point, allowing me to assign multiple tags to an object at once. Now I want to create a method that will get all of the objects in the scene, filter them by the tag, and then return it as an array. The null reference exception refers to line 41 of the Tag.cs script. How do I fix this?
Tags.cs file
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class Tags : MonoBehaviour
{
public string[] startTags;
private string[] tags;
private void Start()
{
tags = startTags;
}
public bool FindTag(string search)
{
bool results = false;
for (int i = 0; i < tags.Length; i++)
{
if(search == tags[i])
{
results = true;
break;
}
}
return results;
}
//Find objects by custom script tags
//HERE IS WHERE THE METHOD IS CREATED
public static GameObject[] ObjectsByTag(string search)
{
//Get all objects in scene
GameObject[] allObjects = FindObjectsOfType<GameObject>();
GameObject[] storedObjects = new GameObject[allObjects.Length];
GameObject[] finalObjects;
//Filter
int count = 0;
for (int i = 0; i < allObjects.Length; i++)
{
if (allObjects[i].GetComponent<Tags>().FindTag(search)) //line 41
{
storedObjects[count] = allObjects[i];
count++;
}
}
//Assign final length
finalObjects = new GameObject[count];
//Reassign to final array
for (int i = 0; i < count; i++)
{
finalObjects[i] = storedObjects[i];
}
return finalObjects;
}
}
GameController.cs file (How it is being used
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class GameController : MonoBehaviour
{
//SCREEN START
//Get Screen Size
private float sHeight;
private float sWidth;
//Intended Screen Size
readonly float iH = 695f;
readonly float iW = 1540f;
//Convert
private float cH;
private float cW;
public float ConvertedHeight => cH;
public float ConvertedWidth => cW;
//SCREEN END
//MOUSE CAM START
//mousePostion
private float mX;
private float mZ;
public float MouseX => mX;
public float MouseZ => mZ;
//MOUSE CAM END
//EnemySpeedModifier
private float esm;
public float ESM
{
get { return esm; }
set { esm = value; }
}
//GameOver
private bool gameOver = false;
public bool GameOver
{
get { return gameOver; }
set { gameOver = value; }
}
//game speed
public float speed;
/*
//projectile list
private GameObject[] projectiles;
public GameObject[] Projectiles()
{
return projectiles;
}
public void Projectiles(GameObject value)
{
GameObject[] tempArray = projectiles;
tempArray[projectiles.Length] = value;
projectiles = tempArray;
Debug.Log("Projectile Count: " + projectiles.Length);
}
*/
//HERE IS WHERE IT IS USED
public GameObject[] ProjectilesInScene
{
get
{
return Tags.ObjectsByTag("projectile");
}
}
// Start is called before the first frame update
void Start()
{
//CONVERT SCREEN SIZES START
sHeight = Screen.height;
sWidth = Screen.width;
cH = iH / sHeight;
cW = iW / sWidth;
//CONVERT SCREEN SIZES END
}
// Update is called once per frame
void Update()
{
if (gameOver)
{
speed /= 1 + 0.5f * Time.deltaTime;
}
//Update mose position
mX = Input.mousePosition.x;
mZ = Input.mousePosition.y;
}
}
It seems that not all of your GameObject objects have the Tags component. Per the GameObject.GetComponent documentation
Returns the component of Type type if the game object has one attached, null if it doesn't.
If you know that some objects won't have the Tags component, your line 41 can use a simple null conditional operator:
if (allObjects[i].GetComponent<Tags>()?.FindTag(search) == true)
{
...
}
Note the ? after GetComponent<Tags>().

How to scale only one game object in the AR scene in Unity?

so I have like 5 game object in my scene but I only scale each of them separately. However when I try to do that all of them start scaling simultaneously. Also, I have a placement indicator that would be used to instantiate the object on the plane. It seems that instead of the object itself, the placement indicator is the one that gets scaled. How should I fix that?
I have tried deactivating the placement indicator but did not work.
Here is the code for instantiating objects:
I limited the obj number to 5.
I use this script instead of the usual "PlaceonPlane" script.
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.XR.ARFoundation;
using UnityEngine.Experimental.XR;
using UnityEngine.UI;
using UnityEngine.XR.ARSubsystems;
public class ARTaptoPlaceObject : MonoBehaviour
{
private ARSessionOrigin arOrigin;
GameObject spawnedobj;
public GameObject placementIndicator;
private ARRaycastManager arRaycast;
public Pose placementPose;
public UIContoller sc;
public bool placementPoseIsValid = false;
private int count;
private string valu;
string prefabs;
void Start()
{
arOrigin = FindObjectOfType<ARSessionOrigin>();
arRaycast = FindObjectOfType<ARRaycastManager>();
count = 0;
}
// Update is called once per frame
void Update()
{
UpdatePlacementPose();
UpdatePlacementIndicator();
for (var i = 0; i < Input.touchCount; ++i)
{
if (Input.GetTouch(i).phase == TouchPhase.Began)
{
if (placementPoseIsValid && Input.GetTouch(i).tapCount == 2)
{
PlaceObject();
}
}
}
}
public void PlaceObject()
{
if (count <= 4)
{
if (sc.objectToPlace != null)
{
spawnedobj = Instantiate(sc.objectToPlace, placementPose.position, placementPose.rotation);
arOrigin.MakeContentAppearAt(spawnedobj.transform, spawnedobj.transform.position, spawnedobj.transform.rotation);
count++;
}
}
else
{
placementIndicator.SetActive(false);
}
}
private void UpdatePlacementIndicator()
{
if (placementPoseIsValid && count <= 4 && sc.active == false)
{
placementIndicator.SetActive(true);
placementIndicator.transform.SetPositionAndRotation(placementPose.position, placementPose.rotation);
}
else
{
placementIndicator.SetActive(false);
}
}
private void UpdatePlacementPose()
{
var screenCenter = Camera.current.ViewportToScreenPoint(new Vector3(0.5f, 0.5f));
var hits = new List<ARRaycastHit>();
arRaycast.Raycast(screenCenter, hits, UnityEngine.XR.ARSubsystems.TrackableType.Planes);
placementPoseIsValid = hits.Count > 0;
if (placementPoseIsValid)
{
placementPose = hits[0].pose;
var cameraForward = Camera.current.transform.forward;
var cameraBearing = new Vector3(cameraForward.x, 0, cameraForward.z).normalized;
placementPose.rotation = Quaternion.LookRotation(cameraBearing);
}
}
}
and here is the Scaler script that's attached to the button that would scale the object.
public class Scaler : MonoBehaviour
{
public UIContoller uc;
public ARTaptoPlaceObject ap;
private GameObject ReferenceToScale;
public void OnValueChange()
{
ReferenceToScale = (UnityEngine.GameObject)Resources.Load(uc.s_count, typeof(GameObject));
Vector3 t = ReferenceToScale.transform.localScale;
Vector3 scaleValue = t * 1.1f;
ReferenceToScale.transform.localScale = scaleValue;
}
Also the "objectToPlace" itself is in the "UI.Controller" script as I could not view it in the scene when it was in the "ARTaptoPlace" script

change radar image based on button click

I could do with some help. I have a radar which displays objects. I'm trying to introduce a radar scan feature so that when a button is clicked the image on the radar is updated based on the tag of the object. My code has no errors but I can't get it to work and was hoping someone here could spot what the problem is. Thanks!!!!
RadarScan Script
public class RadarScan : MonoBehaviour {
public Image RadarImageToChange;
public void ChangeImage(Image UpdateImage)
{
if(gameObject.tag == "Enemy")
{
UpdateImage = RadarImageToChange;
}
}
Radar Script
public class RadarObject
{
public Image icon { get; set; }
public GameObject owner { get; set; }
}
public class Radar : MonoBehaviour {
public Transform playerPos; //position of player
float mapScale = 0.1f; //scale radar size
public static List<RadarObject> radObjects = new List<RadarObject>();
//Registers Object to the radar
public static void RegisterRadarObject(GameObject o, Image i)
{
Image image = Instantiate(i);
radObjects.Add(new RadarObject() { owner = o, icon = image }); //adds to List
}
//It loops through the list looking for the owner existing in the list, when it finds the owner is detroys the icon
public static void RemoveRadarObject(GameObject o)
{
//New list for destroyed objects
List<RadarObject> newList = new List<RadarObject>();
for (int i = 0; i < radObjects.Count; i++)
{
if (radObjects[i].owner == o)
{
Destroy(radObjects[i].icon);
continue;
}
else
newList.Add(radObjects[i]);
}
radObjects.RemoveRange(0, radObjects.Count);
radObjects.AddRange(newList);
}
void DrawRadarDots()
{
//loops through the list and for each Object it gets the owners transform position and determins the difference between it's
//position and the players position, does calculations on the angle and distance and position on a circle using polar equations.
foreach (RadarObject ro in radObjects)
{
Vector3 radarPos = (ro.owner.transform.position - playerPos.position);
float distToObject = Vector3.Distance(playerPos.position, ro.owner.transform.position) * mapScale;
float deltay = Mathf.Atan2(radarPos.x, radarPos.z) * Mathf.Rad2Deg - 270 - playerPos.eulerAngles.y;
radarPos.x = distToObject * Mathf.Cos(deltay * Mathf.Deg2Rad) * -1;
radarPos.z = distToObject * Mathf.Sin(deltay * Mathf.Deg2Rad);
//grabs icon of players objects and make it a child of panel and set it's postion based on radarPos.x and radarPos.z
ro.icon.transform.SetParent(this.transform);
ro.icon.transform.position = new Vector3(radarPos.x, radarPos.z, 0) + this.transform.position;
}
}
//Update is called once per frame
void Update ()
{
DrawRadarDots();
}
}
MakeRadarObject Script
public class MakeRadarObject : MonoBehaviour {
public Image image;
// Use this for initialization
void Start () {
Radar.RegisterRadarObject(this.gameObject, image);
}
void OnDestroy()
{
Radar.RemoveRadarObject(this.gameObject);
}
}
You aren't applying the Image to your gameobject, only to a variable named UpdateImage. You need to get the image component of your gameobject and then assign the new image to it. You will also need to change the Image to the form of a Sprite for this to work.
public Sprite RadarImageToChange;
public void ChangeImage(Sprite UpdateImage)
{
if(gameObject.tag == "Enemy")
{
gameObject.GetComponent<Image>().sprite = RadarImageToChange;
}
}

Stop audio when is playing another one in Unity3D

How can I stop one audio when is playing another one?
public AudioSource audioSource;
public AudioClip audioClip;
private string[,] levelWords;
//public static string glupiaZmienna;
public Text txt;
public void OnPointerClick(PointerEventData eventData)
{
Debug.Log(gameObject.name);
WordsLoader w = new WordsLoader();
levelWords = w.wLevelWords;
SetCategoryButton scb = new SetCategoryButton();
int a = scb.Kategoria;
zupa();
AudioSource audio = gameObject.AddComponent<AudioSource>();
audio.Stop();
audio.clip = (AudioClip)Resources.Load(a+ "/" + WordsLoader.language + "/Sounds/" + gameObject.name);
audio.Play();
// txt = gameObject.GetComponent<Text>();
}
public void zupa()
{ WordsLoader w = new WordsLoader();
SetCategoryButton scb = new SetCategoryButton();
int a = scb.Kategoria;
print(w.wHowManyWords);
for (int i = 0; i < w.wHowManyWords; i++)
{
print("jestem tu");
if (levelWords[i, 0] == gameObject.name)
{
txt = GameObject.Find("Text").GetComponent<Text>(); ;
txt.text = levelWords[i, 2];
}
}
}
This code is in a script which start playing a sound after click a button. I would like stop earlier started audio.
EDIT: Answer has to change a-lot due to comments left under it. It makes sense to overwrite everything.
You need a reference that points to all the AudioSource. Store every ClickAction instance in a List, in another script(PlayerManager). Before you play the audio, loop through that List and stop playing it if it is playing. After that, you can then play your audio.
Also, you currently call Resources.Load each time you want to play the Audio. This is not good in terms of performance. You should do this once in the Start() function, then play it when it is needed. Also, your gameObject.AddComponent<AudioSource>() code should only be called once too.
Your new ClickAction script:
public class ClickAction : MonoBehaviour
{
public AudioSource audioSource;
public AudioClip audioClip;
private string[,] levelWords;
//public static string glupiaZmienna;
public Text txt;
public AudioSource clickAudioSource;
PlayerManager playerManager = null;
void Start()
{
//Get PlayerManager from PlayerManager GameObject
playerManager = GameObject.Find("PlayerManager").GetComponent<PlayerManager>();
clickAudioSource = gameObject.AddComponent<AudioSource>();
clickAudioSource.clip = (AudioClip)Resources.Load(a + "/" + WordsLoader.language + "/Sounds/" + gameObject.name);
//Register this Script to the PlayerManager
playerManager.addClickAction(this);
}
public void OnPointerClick(PointerEventData eventData)
{
Debug.Log(gameObject.name);
WordsLoader w = new WordsLoader();
levelWords = w.wLevelWords;
SetCategoryButton scb = new SetCategoryButton();
int a = scb.Kategoria;
zupa();
//Play Audio
playAudio();
// txt = gameObject.GetComponent<Text>();
}
private void playAudio()
{
playerManager.playAudio(clickAudioSource);
}
public void stopAudio()
{
//If not null and playing then stop audio
if (clickAudioSource != null && clickAudioSource.isPlaying)
{
clickAudioSource.Stop();
}
}
public void zupa()
{
WordsLoader w = new WordsLoader();
SetCategoryButton scb = new SetCategoryButton();
int a = scb.Kategoria;
print(w.wHowManyWords);
for (int i = 0; i < w.wHowManyWords; i++)
{
print("jestem tu");
if (levelWords[i, 0] == gameObject.name)
{
txt = GameObject.Find("Text").GetComponent<Text>(); ;
txt.text = levelWords[i, 2];
}
}
}
void OnDisable()
{
//Un-Register this Script from the PlayerManager when it is destroyed
playerManager.removeClickAction(this);
}
}
PlayerManager script. Create a GameObject called PlayerManager, then attach the script below to it:
public class PlayerManager : MonoBehaviour
{
List<ClickAction> clickActions = new List<ClickAction>();
public void addClickAction(ClickAction clickActionToAdd)
{
//Adds clickActionToAdd if it does not exist in the List already
if (!clickActions.Contains(clickActionToAdd))
{
clickActions.Add(clickActionToAdd);
}
}
public void removeClickAction(ClickAction clickActionToAdd)
{
//Removes clickActionToAdd if it exist in the List
if (clickActions.Contains(clickActionToAdd))
{
clickActions.Add(clickActionToAdd);
}
}
public bool clickActionExist(ClickAction clickActionToAdd)
{
//Removes clickActionToAdd if it exist in the List
return clickActions.Contains(clickActionToAdd);
}
public void playAudio(AudioSource audSource)
{
//Stop Other Audio
stopAllClickActionAudio();
//Now, play the one that was passed in
audSource.Play();
}
void stopAllClickActionAudio()
{
//Stop Audio on every ClickAction script
for (int i = 0; i < clickActions.Count; i++)
{
clickActions[i].stopAudio();
}
}
}
When the playAudio function is called, it will stop all audio from AudioSource that is stored in the List, then it will play the current AudioSource that is passed into it.

Categories