Upload 4 files

README.md

@@ -4,16 +4,16 @@ library_name: unity-sentis
 pipeline_tag: image-classification
 ---
 
-## MobileNet V2 in Unity Sentis Format (Version 1.3.0-pre.3*)
-*Version 1.3.0 Sentis files are not compatible with Sentis 1.4.0 and need to be recreated/downloaded
+## MobileNet V2 in Unity Sentis Format (Version 1.4.0-pre.2*)
+*Version 1.3.0 Sentis files are not compatible with 1.4.0 and need to be recreated/downloaded
 
 This is a small image classification model that works in Unity 2023. It is based on [MobileNet V2](https://arxiv.org/abs/1801.04381)
 
 ## How to Use
 * Create a new scene in Unity 2023
-* Install `com.unity.sentis` version `1.3.0-pre.3` from the package manager
+* Install `com.unity.sentis` version `1.4.0-pre.2` from the package manager
 * Add the C# script to the Main Camera
-* Put the `mobilenet_v2.sentis` model in the `Assets/StreamingAssets` folder
+* Drag the `mobilenet_v2.sentis` model onto the `modelAsset `field
 * Drag the `class_desc.txt` on to the `labelsAsset` field
 * Drag one of the sample images on to the inputImage field in the inspector.
 * Press play and the result of the prediction will print to the console window.

RunMobileNet.cs

@@ -1,7 +1,8 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using Unity.Sentis;
 using UnityEngine;
-
+using System.IO;
+using FF = Unity.Sentis.Functional;
 /*
  *  MovileNetV2 Inference Script
  *  ============================
@@ -18,6 +19,9 @@ using UnityEngine;
 
 public class RunMobileNet : MonoBehaviour
 {
+    //draw the sentis file here:
+    public ModelAsset modelAsset;
+
     const string modelName = "mobilenet_v2.sentis";
 
     //The image to classify here:
@@ -26,32 +30,42 @@ public class RunMobileNet : MonoBehaviour
     //Link class_desc.txt here:
     public TextAsset labelsAsset;
 
-    //All images are resized to these values:
+    //All images are resized to these values to go into the model
     const int imageHeight = 224;
     const int imageWidth = 224;
 
     const BackendType backend = BackendType.GPUCompute;
-    private Model model;
+
     private IWorker engine;
     private string[] labels;
 
-    static Ops ops;
-    ITensorAllocator allocator;
+    //Used to normalise the input RGB values
+    TensorFloat mulRGB = new TensorFloat(new TensorShape(1, 3, 1, 1), new float[] { 1 / 0.229f, 1 / 0.224f, 1 / 0.225f });
+    TensorFloat shiftRGB = new TensorFloat(new TensorShape(1, 3, 1, 1), new float[] { 0.485f, 0.456f, 0.406f });
 
     void Start()
     {
-        //These are used for tensor operations
-        allocator = new TensorCachingAllocator();
-        ops = WorkerFactory.CreateOps(backend, allocator);
 
         //Parse neural net labels
         labels = labelsAsset.text.Split('\n');
 
-        //Load model
-        model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName);
+        //Load model from file or asset
+        //var model = ModelLoader.Load(Path.Join(Application.streamingAssetsPath, modelName));
+        var model = ModelLoader.Load(modelAsset);
+
+        //We modify the model to normalise the input RGB values and select the highest prediction
+        //probability and item number
+        var model2 = FF.Compile(
+            input =>
+            {
+                var probability = model.Forward(NormaliseRGB(input))[0];
+                return (FF.ReduceMax(probability, 1), FF.ArgMax(probability, 1));
+            },
+            model.inputs[0]
+        );
 
         //Setup the engine to run the model
-        engine = WorkerFactory.CreateWorker(backend, model);
+        engine = WorkerFactory.CreateWorker(backend, model2);
 
         //Execute inference
         ExecuteML();
@@ -60,51 +74,39 @@ public class RunMobileNet : MonoBehaviour
     public void ExecuteML()
     {
         //Preprocess image for input
-        using var rawinput = TextureConverter.ToTensor(inputImage, 224, 224, 3);
-        using var input = Normalise(rawinput);
+        using var input = TextureConverter.ToTensor(inputImage, imageWidth, imageHeight, 3);
         
         //Execute neural net
         engine.Execute(input);
 
         //Read output tensor
-        var output = engine.PeekOutput() as TensorFloat;
-        var argmax = ops.ArgMax(output, 1, false);
-        argmax.MakeReadable();
+        var probability = engine.PeekOutput("output_0") as TensorFloat;
+        var item = engine.PeekOutput("output_1") as TensorInt;
+        item.CompleteOperationsAndDownload();
+        probability.CompleteOperationsAndDownload();
 
         //Select the best output class and print the results
-        var res = argmax[0];
-        var label = labels[res];
-
-        output.MakeReadable();
-        var accuracy = output[res];
+        var ID = item[0];
+        var accuracy = probability[0];
 
         //The result is output to the console window
         int percent = Mathf.FloorToInt(accuracy * 100f + 0.5f);
-        Debug.Log($"{label} {percent}﹪");
+        Debug.Log($"Prediction: {labels[ID]} {percent}﹪");
 
         //Clean memory
         Resources.UnloadUnusedAssets();
     }
 
     //This scales and shifts the RGB values for input into the model
-    TensorFloat Normalise(TensorFloat image)
+    FunctionalTensor NormaliseRGB(FunctionalTensor image)
     {
-        using var M = new TensorFloat(new TensorShape(1, 3, 1, 1), new float[]
-        {
-           1/0.229f, 1/0.224f, 1/0.225f
-        });
-        using var P = new TensorFloat(new TensorShape(1, 3, 1, 1), new float[]
-        {
-            0.485f, 0.456f, 0.406f
-        });
-        using var image2 = ops.Sub(image, P);
-        return ops.Mul(image2, M);
+        return (image - FunctionalTensor.FromTensor(shiftRGB)) * FunctionalTensor.FromTensor(mulRGB);
     }
     
     private void OnDestroy()
     {
-        engine?.Dispose();
-        ops?.Dispose();
-        allocator?.Dispose();
+        mulRGB?.Dispose();
+        shiftRGB?.Dispose();
+        engine?.Dispose();      
     }
 }

info.json

@@ -9,6 +9,6 @@
         "class_desc.txt"
     ],
     "version":[
-         "1.3.0-pre.3"
+         "1.4.0"
     ]
 }

mobilenet_v2.sentis

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81e2e51fad533a32b721454cad79341da8457f8b361f2868319eac93d6b9c5e7
-size 14046825
+oid sha256:907d42cf325f7d2457b8ccdd12fabb5bea882d2757c3bb5bc57042e5ec6533bc
+size 13989036