Unity + OpenCV OpticalFlowPyrLK + Visual Effect Graph

1. Webcam

OpticalFlowPyrLK  함수를 쓰기위해서 현재 프레임과 이전프레임이 필요

using System;
using OpenCvSharp;
using UnityEngine;

public class WebCam : MonoBehaviour
{
    public Material mat;
    WebCamTexture webCamTexture;
    
    private OpenCVUnity Convert = new OpenCVUnity();
    IplImage prev;
    IplImage curr;
    IplImage dst;
    private Texture2D dstTexture; // 계산 결과를 담을 텍스쳐

    public CvPoint2D32f[] cornersPrev;
    public CvPoint2D32f[] cornersCurr;
    public sbyte[] status;

    public BufferGenerator bufferGenerator;

    private void Start()
    {
        WebCamDevice device = WebCamTexture.devices[0];

        webCamTexture = new WebCamTexture(device.name);
        webCamTexture.Play();

        dstTexture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);

        curr = WebcamTex2Ipl(webCamTexture);
    }

    private void Update()
    {
        if (curr != null) prev = curr;
        if (webCamTexture.didUpdateThisFrame)
        {
            curr = WebcamTex2Ipl(webCamTexture);
            dst = Convert.OpticalFlowPyrLK(prev, curr, out cornersPrev, out cornersCurr, out status);

            // 광학흐름 계산 결과 이미지 확인용
            dstTexture.LoadRawTextureData(dst.ImageData, webCamTexture.width * webCamTexture.height * 4);
            dstTexture.Apply();
            mat.mainTexture = dstTexture;
        }

        bufferGenerator.UpdateBuffer();
    }

    IplImage WebcamTex2Ipl(WebCamTexture wct)
    {
        Color32[] camTex32 = wct.GetPixels32();
        CvSize size = new CvSize(wct.width, wct.height);

        return Convert.FlipImage(IplImage.FromPixelData(size, BitDepth.U8, 4, camTex32));
    }

    private void OnDestroy()
    {
        curr.Dispose();
        dst.Dispose();
        prev.Dispose();
    }
}

2. OpenCVSharp2 OpticalFlowPyrLK 함수 사용

 (OpenCVSharp 강의의 예제 코드 활용)

https://076923.github.io/posts/C-opencv-59/

public IplImage OpticalFlowPyrLK(IplImage previous, IplImage current, out CvPoint2D32f[] corners, out CvPoint2D32f[] corners2, out sbyte[] status)

 out 키워드를 써서 코너데이터 배열과 움직임이 있는지 유무를 저장한 배열을 받아옴

3. Graphic 버퍼

버퍼 생성

검출한 corner 데이터와 status 데이터를 버퍼로 변환

 

using System;
using System.Collections;
using System.Collections.Generic;
using OpenCvSharp;
using UnityEditorInternal;
using UnityEngine;

public class BufferGenerator : MonoBehaviour
{
    public WebCam webcam;

    public GraphicsBuffer cornersPrev;
    public GraphicsBuffer cornersCurr;
    public GraphicsBuffer cornersStatus;

    int cornerCount = 300;

    private void Start()
    {
        cornersPrev = new GraphicsBuffer(GraphicsBuffer.Target.Structured, cornerCount, sizeof(float) * 2);
        cornersCurr = new GraphicsBuffer(GraphicsBuffer.Target.Structured, cornerCount, sizeof(float) * 2);
        cornersStatus = new GraphicsBuffer(GraphicsBuffer.Target.Structured, cornerCount, sizeof(int));
    }

    public void UpdateBuffer()
    {
        var prev = new Vector2[cornerCount];
        var curr = new Vector2[cornerCount];
        var status = new int[cornerCount];

        for (int i = 0; i < cornerCount; i++)
        {
            prev[i] = new Vector2(webcam.cornersPrev[i].X, webcam.cornersPrev[i].Y);
            curr[i] = new Vector2(webcam.cornersCurr[i].X, webcam.cornersCurr[i].Y);
            status[i] = (int) webcam.status[i];
        }

        cornersPrev.SetData(prev);
        cornersCurr.SetData(curr);
        cornersStatus.SetData(status);
    }

    public void OnDestroy()
    {
        if (cornersPrev != null) cornersPrev.Release();
        if (cornersCurr != null) cornersCurr.Release();
        if (cornersStatus != null) cornersStatus.Release();
    }
}

 

vfxgraph의 프로퍼티 바인딩 코드를 작성

using UnityEngine;
using UnityEngine.VFX;
using UnityEngine.VFX.Utility;

[VFXBinder(nameof(BufferBinder))]
public class BufferBinder : VFXBinderBase
{
    public BufferGenerator bufferGenerator;

    [SerializeField, VFXPropertyBinding(nameof(UnityEngine.GraphicsBuffer))]
    private ExposedProperty corners1BufferName = "cornersPrev";

    [SerializeField, VFXPropertyBinding(nameof(UnityEngine.GraphicsBuffer))]
    private ExposedProperty corners2BufferName = "cornersCurr";

    [SerializeField, VFXPropertyBinding(nameof(UnityEngine.GraphicsBuffer))]
    private ExposedProperty statusBufferName = "statusBuffer";


    [SerializeField, VFXPropertyBinding(nameof(UnityEngine.Vector2))]
    private ExposedProperty webcamResolutionName = "webcamResolution";

    [SerializeField, VFXPropertyBinding(nameof(System.Single))]
    private ExposedProperty reductionMagnificationName = "reductionMagnification";

    [SerializeField, VFXPropertyBinding(nameof(UnityEngine.Vector3))]
    private ExposedProperty displayPositionName = "displayPosition";


    public Vector2 webcamResolution;
    public float reductionMagnification;
    public Transform displayPosition;

    public override bool IsValid(VisualEffect component)
    {
        return bufferGenerator != null
               && component.HasGraphicsBuffer(corners1BufferName)
               && component.HasGraphicsBuffer(corners2BufferName)
               && component.HasGraphicsBuffer(statusBufferName)
               && component.HasVector2(webcamResolutionName)
               && component.HasFloat(reductionMagnificationName)
               && component.HasVector3(displayPositionName);
    }

    public override void UpdateBinding(VisualEffect component)
    {
        if (!Application.isPlaying) return;
        component.SetGraphicsBuffer(corners1BufferName, bufferGenerator.cornersPrev);
        component.SetGraphicsBuffer(corners2BufferName, bufferGenerator.cornersCurr);
        component.SetGraphicsBuffer(statusBufferName, bufferGenerator.cornersStatus);
        component.SetVector2(webcamResolutionName, webcamResolution);
        component.SetFloat(reductionMagnificationName, reductionMagnification);
        component.SetVector3(displayPositionName, displayPosition.position);
    }
}

4. Visual Effect Graph

서브그래프 내용

결과