modified the ppo with heuristic and Pong example

Author: tcmxx

Assets/UnityTensorflow/Common/Utils/DataBuffer.cs

@@ -28,7 +28,7 @@ public DataInfo(string name, Type type, int[] dimension)
     }
 
     [Serializable]
-    protected struct DataContainer
+    protected class DataContainer
     {
 
         public DataInfo info;
@@ -45,10 +45,7 @@ public DataContainer(DataInfo info, int reservedSize)
             this.info = info;
             dataList = Array.CreateInstance(info.type, (new int[] { reservedSize }).Concat(info.dimension).ToArray());
         }
-
-
-
-
+        
 
         public void IncreaseArraySize(int sizeToAdd)
         {
@@ -57,7 +54,7 @@ public void IncreaseArraySize(int sizeToAdd)
 
             var newArray = Array.CreateInstance(info.type, (new int[] { dataList.GetLength(0) + sizeToAdd }).Concat(info.dimension).ToArray());
             int typeSize = Marshal.SizeOf(info.type);
-            Buffer.BlockCopy(dataList, 0, newArray, 0, dataList.Length * info.unitLength * typeSize);
+            Buffer.BlockCopy(dataList, 0, newArray, 0, dataList.Length  * typeSize);
             dataList = newArray;
 
         }
@@ -77,6 +74,19 @@ public int CurrentSize()
     public int CurrentCount { get; private set; } = 0;
 
 
+    public DataBuffer(params DataInfo[] dataInfos)
+    {
+
+        MaxCount = 0;
+        dataset = new Dictionary<string, DataContainer>();
+
+
+        foreach (var i in dataInfos)
+        {
+            Debug.Assert(!dataset.ContainsKey(i.name));
+            dataset[i.name] = new DataContainer(i);
+        }
+    }
 
     public DataBuffer(int maxSize, params DataInfo[] dataInfos)
     {
@@ -88,7 +98,10 @@ public DataBuffer(int maxSize, params DataInfo[] dataInfos)
         foreach (var i in dataInfos)
         {
             Debug.Assert(!dataset.ContainsKey(i.name));
-            dataset[i.name] = new DataContainer(i, maxSize);
+            if(MaxCount > 0)
+                dataset[i.name] = new DataContainer(i, maxSize);
+            else
+                dataset[i.name] = new DataContainer(i);
         }
     }
 
@@ -151,10 +164,12 @@ public void AddData(params ValueTuple<string, Array>[] data)
         foreach (var k in data)
         {
             //resize the data container if needed
-            while(CurrentCount > dataset[k.Item1].CurrentSize())
-            {
-                dataset[k.Item1].IncreaseArraySize(dataset[k.Item1].CurrentSize());
-            }
+            int currentSize = dataset[k.Item1].CurrentSize();
+            if(CurrentCount > currentSize * 2)
+                dataset[k.Item1].IncreaseArraySize(CurrentCount - currentSize);
+            else if(CurrentCount > currentSize)
+                dataset[k.Item1].IncreaseArraySize(currentSize);
+            
 
             DataContainer dd = dataset[k.Item1];
             int typeSize = Marshal.SizeOf(dd.info.type);

Assets/UnityTensorflow/Examples/3DBall/3DBallNE.unity

@@ -691,12 +691,12 @@ MonoBehaviour:
     vectorActionSpaceType: 1
   brainType: 4
   CoreBrains:
-  - {fileID: 1430763440}
-  - {fileID: 1537357350}
-  - {fileID: 1709965260}
-  - {fileID: 1206030199}
-  - {fileID: 891779459}
-  instanceID: 72450
+  - {fileID: 1655782775}
+  - {fileID: 1783173004}
+  - {fileID: 1491512661}
+  - {fileID: 1499967835}
+  - {fileID: 1909405677}
+  instanceID: 24670
 --- !u!1001 &764818074
 Prefab:
   m_ObjectHideFlags: 0
@@ -817,42 +817,6 @@ Prefab:
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: ff026d63a00abdc48ad6ddcff89aba04, type: 2}
   m_IsPrefabParent: 0
---- !u!114 &891779459
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_PrefabParentObject: {fileID: 0}
-  m_PrefabInternal: {fileID: 0}
-  m_GameObject: {fileID: 0}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 6c70bae3b443c3743a743c3b2837676b, type: 3}
-  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
-  m_EditorClassIdentifier: 
-  brain: {fileID: 667765197}
-  trainer: {fileID: 580003488}
---- !u!114 &1206030199
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_PrefabParentObject: {fileID: 0}
-  m_PrefabInternal: {fileID: 0}
-  m_GameObject: {fileID: 0}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 8b23992c8eb17439887f5e944bf04a40, type: 3}
-  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
-  m_EditorClassIdentifier: 
-  broadcast: 1
-  graphModel: {fileID: 0}
-  graphScope: 
-  graphPlaceholders: []
-  BatchSizePlaceholderName: batch_size
-  VectorObservationPlacholderName: vector_observation
-  RecurrentInPlaceholderName: recurrent_in
-  RecurrentOutPlaceholderName: recurrent_out
-  VisualObservationPlaceholderName: []
-  ActionPlaceholderName: action
-  PreviousActionPlaceholderName: prev_action
-  brain: {fileID: 0}
 --- !u!1 &1397918840
 GameObject:
   m_ObjectHideFlags: 0
@@ -918,23 +882,6 @@ Transform:
   m_Father: {fileID: 0}
   m_RootOrder: 0
   m_LocalEulerAnglesHint: {x: 20, y: 0, z: 0}
---- !u!114 &1430763440
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_PrefabParentObject: {fileID: 0}
-  m_PrefabInternal: {fileID: 0}
-  m_GameObject: {fileID: 0}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 41e9bda8f3cf1492fa74926a530f6f70, type: 3}
-  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
-  m_EditorClassIdentifier: 
-  broadcast: 1
-  keyContinuousPlayerActions: []
-  axisContinuousPlayerActions: []
-  discretePlayerActions: []
-  defaultAction: 0
-  brain: {fileID: 667765197}
 --- !u!1001 &1450507641
 Prefab:
   m_ObjectHideFlags: 0
@@ -1026,18 +973,40 @@ Transform:
   m_Father: {fileID: 1583402088}
   m_RootOrder: 2
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
---- !u!114 &1537357350
+--- !u!114 &1491512661
 MonoBehaviour:
   m_ObjectHideFlags: 0
   m_PrefabParentObject: {fileID: 0}
   m_PrefabInternal: {fileID: 0}
   m_GameObject: {fileID: 0}
   m_Enabled: 1
   m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 943466ab374444748a364f9d6c3e2fe2, type: 3}
-  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
+  m_Script: {fileID: 11500000, guid: 35813a1be64e144f887d7d5f15b963fa, type: 3}
+  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
+  m_EditorClassIdentifier: 
+  brain: {fileID: 0}
+--- !u!114 &1499967835
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 8b23992c8eb17439887f5e944bf04a40, type: 3}
+  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
   m_EditorClassIdentifier: 
   broadcast: 1
+  graphModel: {fileID: 0}
+  graphScope: 
+  graphPlaceholders: []
+  BatchSizePlaceholderName: batch_size
+  VectorObservationPlacholderName: vector_observation
+  RecurrentInPlaceholderName: recurrent_in
+  RecurrentOutPlaceholderName: recurrent_out
+  VisualObservationPlaceholderName: []
+  ActionPlaceholderName: action
+  PreviousActionPlaceholderName: prev_action
   brain: {fileID: 0}
 --- !u!1 &1583402087
 GameObject:
@@ -1101,18 +1070,23 @@ MonoBehaviour:
   resetParameters:
     resetParameters: []
   bounceTreshold: 2
---- !u!114 &1709965260
+--- !u!114 &1655782775
 MonoBehaviour:
   m_ObjectHideFlags: 0
   m_PrefabParentObject: {fileID: 0}
   m_PrefabInternal: {fileID: 0}
   m_GameObject: {fileID: 0}
   m_Enabled: 1
   m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 35813a1be64e144f887d7d5f15b963fa, type: 3}
-  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
+  m_Script: {fileID: 11500000, guid: 41e9bda8f3cf1492fa74926a530f6f70, type: 3}
+  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
   m_EditorClassIdentifier: 
-  brain: {fileID: 0}
+  broadcast: 1
+  keyContinuousPlayerActions: []
+  axisContinuousPlayerActions: []
+  discretePlayerActions: []
+  defaultAction: 0
+  brain: {fileID: 667765197}
 --- !u!1 &1746325439
 GameObject:
   m_ObjectHideFlags: 0
@@ -1235,6 +1209,19 @@ Prefab:
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: ff026d63a00abdc48ad6ddcff89aba04, type: 2}
   m_IsPrefabParent: 0
+--- !u!114 &1783173004
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 943466ab374444748a364f9d6c3e2fe2, type: 3}
+  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
+  m_EditorClassIdentifier: 
+  broadcast: 1
+  brain: {fileID: 0}
 --- !u!1001 &1825513289
 Prefab:
   m_ObjectHideFlags: 0
@@ -1295,6 +1282,19 @@ Prefab:
     m_RemovedComponents: []
   m_ParentPrefab: {fileID: 100100000, guid: ff026d63a00abdc48ad6ddcff89aba04, type: 2}
   m_IsPrefabParent: 0
+--- !u!114 &1909405677
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 6c70bae3b443c3743a743c3b2837676b, type: 3}
+  m_Name: (Clone)(Clone)(Clone)(Clone)(Clone)(Clone)(Clone)
+  m_EditorClassIdentifier: 
+  brain: {fileID: 667765197}
+  trainer: {fileID: 580003488}
 --- !u!1 &2063801192
 GameObject:
   m_ObjectHideFlags: 0

Assets/UnityTensorflow/Examples/3DBall/3DBallNECheckpoint.bytes

@@ -0,0 +1,28 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!114 &11400000
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_PrefabParentObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: b3b2cffc6f799a841a0dc06b2a622d21, type: 3}
+  m_Name: PongPPOHeuristicTrainerParams
+  m_EditorClassIdentifier: 
+  learningRate: 0.004
+  maxTotalSteps: 2000000
+  lossLogInterval: 1
+  saveModelInterval: 50000
+  rewardDiscountFactor: 0.99
+  rewardGAEFactor: 0.98
+  valueLossWeight: 1
+  timeHorizon: 1000
+  entropyLossWeight: 0.1
+  clipEpsilon: 0.2
+  batchSize: 256
+  bufferSizeForTrain: 8192
+  numEpochPerTrain: 10
+  useHeuristicChance: 1
+  rewardLogInterval: 50

Assets/UnityTensorflow/Examples/3DBall/3DBallNECheckpoint_NEData.bytes

@@ -11,18 +11,18 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: b3b2cffc6f799a841a0dc06b2a622d21, type: 3}
   m_Name: PongPPOTrainerParams
   m_EditorClassIdentifier: 
-  maxTotalSteps: 100000000
+  learningRate: 0.004
+  maxTotalSteps: 1000000
+  lossLogInterval: 1
+  saveModelInterval: 50000
   rewardDiscountFactor: 0.99
-  rewardGAEFactor: 0.99
+  rewardGAEFactor: 0.98
   valueLossWeight: 1
-  entroyLossWeight: 0
+  timeHorizon: 1000
+  entropyLossWeight: 0.1
   clipEpsilon: 0.2
   batchSize: 256
   bufferSizeForTrain: 8192
   numEpochPerTrain: 10
-  heuristicBufferSize: 100000
-  extraBatchTFromHeuristicBuffer: 5
-  learningRate: 0.01
-  lossLogInterval: 1
+  useHeuristicChance: 1
   rewardLogInterval: 50
-  saveModelInterval: 50000