mantis 8518: Yengine; we can't wait for GC (worse finalizers) to count released memory of some local variables, so add a pseudo free; fix memory account on timeslice rentry; change the folder for the debug IL files; fix memory usage on reset. This changes will only take effect on new compiles

OpenSim/Region/ScriptEngine/YEngine/MMRScriptCodeGen.cs

@@ -197,7 +197,7 @@ namespace OpenSim.Region.ScriptEngine.Yengine
         public CallLabel openCallLabel = null;  // only one call label can be open at a time
                                                 // - the call label is open from the time of CallPre() until corresponding CallPost()
                                                 // - so no non-trivial pushes/pops etc allowed between a CallPre() and a CallPost()
-
+        public List<ScriptMyLocal> HeapLocals = new List<ScriptMyLocal>();
         private ScriptMyILGen _ilGen;
         public ScriptMyILGen ilGen
         {
@@ -1258,6 +1258,7 @@ namespace OpenSim.Region.ScriptEngine.Yengine
              // resume at the correct spot.
             actCallLabels.Clear();
             allCallLabels.Clear();
+            HeapLocals.Clear();
             openCallLabel = null;
 
              // Alloc stack space for local vars.
@@ -1398,8 +1399,6 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             _ilGen = collector.WriteOutAll();
             collector = null;
 
-             // Output code to restore stack frame from stream.
-             // It jumps back to the call labels within the function body.
             List<ScriptMyLocal> activeTemps = null;
             if (!isTrivial)
             {
@@ -1452,11 +1451,34 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             }
 
              // Output the 'real' return opcode.
+             // push return value
             ilGen.MarkLabel(retLabel);
             if (!(curDeclFunc.retType is TokenTypeVoid))
             {
                 ilGen.Emit(curDeclFunc, OpCodes.Ldloc, retValue);
             }
+
+            // pseudo free memory usage
+            foreach (ScriptMyLocal sml in HeapLocals)
+            {
+                Type t = sml.type;
+                if (t == typeof(HeapTrackerList))
+                {
+                    ilGen.Emit(curDeclFunc, OpCodes.Ldloc, sml);
+                    HeapTrackerList.GenFree(curDeclFunc, ilGen);
+                }
+                else if (t == typeof(HeapTrackerString))
+                {
+                    ilGen.Emit(curDeclFunc, OpCodes.Ldloc, sml);
+                    HeapTrackerString.GenFree(curDeclFunc, ilGen);
+                }
+                else if (t == typeof(HeapTrackerObject))
+                {
+                    ilGen.Emit(curDeclFunc, OpCodes.Ldloc, sml);
+                    HeapTrackerObject.GenFree(curDeclFunc, ilGen);
+                }
+            }
+
             ilGen.Emit(curDeclFunc, OpCodes.Ret);
             retLabel = null;
             retValue = null;
@@ -1675,11 +1697,11 @@ namespace OpenSim.Region.ScriptEngine.Yengine
                 if(u != t)
                 {
                     if(t == typeof(HeapTrackerList))
-                        HeapTrackerList.GenPop(curDeclFunc, ilGen);
+                        HeapTrackerList.GenRestore(curDeclFunc, ilGen);
                     if(t == typeof(HeapTrackerObject))
-                        HeapTrackerObject.GenPop(curDeclFunc, ilGen);
+                        HeapTrackerObject.GenRestore(curDeclFunc, ilGen);
                     if(t == typeof(HeapTrackerString))
-                        HeapTrackerString.GenPop(curDeclFunc, ilGen);
+                        HeapTrackerString.GenRestore(curDeclFunc, ilGen);
                 }
                 else
                 {

OpenSim/Region/ScriptEngine/YEngine/MMRScriptCollector.cs

@@ -2611,10 +2611,10 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             // everything required by any blocks it can branch to.
             do
             {
-                this.resolvedSomething = false;
+                resolvedSomething = false;
-                this.resolveSequence++;
+                resolveSequence++;
-                this.ResolveBlock((GraphNodeBlock)firstLin);
+                ResolveBlock((GraphNodeBlock)firstLin);
-            } while(this.resolvedSomething);
+            } while(resolvedSomething);
 
             // Repeat the cutting loops as long as we keep finding stuff.
             bool didSomething;
@@ -2939,7 +2939,7 @@ namespace OpenSim.Region.ScriptEngine.Yengine
                 return;
 
             // So we don't recurse forever on a backward branch.
-            currentBlock.hasBeenResolved = this.resolveSequence;
+            currentBlock.hasBeenResolved = resolveSequence;
 
             // Assume we haven't written any locals yet.
             List<ScriptMyLocal> localsWrittenSoFar = new List<ScriptMyLocal>();
@@ -2975,7 +2975,7 @@ namespace OpenSim.Region.ScriptEngine.Yengine
                                 !currentBlock.localsReadBeforeWritten.Contains(readByNextBlock))
                             {
                                 currentBlock.localsReadBeforeWritten.Add(readByNextBlock);
-                                this.resolvedSomething = true;
+                                resolvedSomething = true;
                             }
                         }
                     }

OpenSim/Region/ScriptEngine/YEngine/MMRScriptCompValu.cs

@@ -1483,7 +1483,8 @@ namespace OpenSim.Region.ScriptEngine.Yengine
         {
             if(type.ToHeapTrackerType() != null)
             {
-                this.localBuilder = scg.ilGen.DeclareLocal(type.ToHeapTrackerType(), name);
+                localBuilder = scg.ilGen.DeclareLocal(type.ToHeapTrackerType(), name);
+                scg.HeapLocals.Add(localBuilder);
                 scg.PushXMRInst();
                 scg.ilGen.Emit(type, OpCodes.Newobj, type.GetHeapTrackerCtor());
                 scg.ilGen.Emit(type, OpCodes.Stloc, localBuilder);
@@ -1547,6 +1548,7 @@ namespace OpenSim.Region.ScriptEngine.Yengine
                 scg.ilGen.Emit(errorAt, OpCodes.Ldloc, localBuilder);
                 scg.ilGen.Emit(errorAt, OpCodes.Ldloc, htpop);
                 type.CallHeapTrackerPopMeth(errorAt, scg.ilGen);
+                scg.HeapLocals.Add(htpop);
             }
             else
             {

OpenSim/Region/ScriptEngine/YEngine/MMRScriptCompile.cs

@@ -130,7 +130,7 @@ namespace OpenSim.Region.ScriptEngine.Yengine
                  // Since we just wrote the .xmrobj file, maybe save disassembly.
                 if (m_Engine.m_ScriptDebugSaveIL)
                 {
-                    string asmFileName = GetScriptFileName (m_ScriptObjCodeKey + ".yasm");
+                    string asmFileName = GetScriptILFileName(m_ScriptObjCodeKey + ".yasm");
 //                    m_log.Debug ("[YEngine]: MMRScriptCompileSaveILGen: saving to " + asmFileName);
                     asmFileWriter = File.CreateText (asmFileName);
                 }

OpenSim/Region/ScriptEngine/YEngine/XMRArray.cs

@@ -285,7 +285,6 @@ namespace OpenSim.Region.ScriptEngine.Yengine
         public void RecvArrayObj(RecvArrayObjDelegate recvObj)
         {
             heapUse = inst.UpdateHeapUse(heapUse, EMPTYHEAP);
-
             // Cause any enumeration to refill the array from the sorted dictionary.
             // Since it is a sorted dictionary, any enumerations will be in the same 
             // order as on the sending side.

OpenSim/Region/ScriptEngine/YEngine/XMRHeapTracker.cs

@@ -58,11 +58,7 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             if(inst == null)
                 throw new ArgumentNullException("inst");
             instance = inst;
-        }
+            usage = 0;
-
-        ~HeapTrackerBase()
-        {
-            usage = instance.UpdateHeapUse(usage, 0);
         }
     }
 
@@ -73,24 +69,33 @@ namespace OpenSim.Region.ScriptEngine.Yengine
     {
         private static FieldInfo listValueField = typeof(HeapTrackerList).GetField("value");
         private static MethodInfo listSaveMethod = typeof(HeapTrackerList).GetMethod("Save");
+        private static MethodInfo listRestoreMethod = typeof(HeapTrackerList).GetMethod("Restore");
+        private static MethodInfo listFreeMethod = typeof(HeapTrackerList).GetMethod("Free");
 
         public LSL_List value;
 
         public HeapTrackerList(XMRInstAbstract inst) : base(inst) {}
 
-        // generate CIL code to pop the value from the CIL stack
+        // generate CIL code to pop the value ie store in value
         //  input:
         //   'this' pointer already pushed on CIL stack
-        //   new value pushed on CIL stack
+        //   new value
         //  output:
-        //   'this' pointer popped from stack
-        //   new value popped from CIL stack
-        //   heap usage updated
         public static void GenPop(Token errorAt, ScriptMyILGen ilGen)
         {
             ilGen.Emit(errorAt, OpCodes.Call, listSaveMethod);
         }
 
+        public static void GenRestore(Token errorAt, ScriptMyILGen ilGen)
+        {
+            ilGen.Emit(errorAt, OpCodes.Call, listRestoreMethod);
+        }
+
+        public static void GenFree(Token errorAt, ScriptMyILGen ilGen)
+        {
+            ilGen.Emit(errorAt, OpCodes.Call, listFreeMethod);
+        }
+
         // generate CIL code to push the value on the CIL stack
         //  input:
         //   'this' pointer already pushed on CIL stack
@@ -106,23 +111,32 @@ namespace OpenSim.Region.ScriptEngine.Yengine
 
         public void Save(LSL_List lis)
         {
-            int newuse = Size(lis);
+            if (lis == null)
-            usage = instance.UpdateHeapUse(usage, newuse);
+                usage = instance.UpdateHeapUse(usage, 0);
+            else
+                usage = instance.UpdateHeapUse(usage, Size(lis));
             value = lis;
         }
 
+        public void Restore(LSL_List lis)
+        {
+            value = lis;
+            if (lis != null)
+                usage = Size(lis);
+            else
+                usage = 0;
+        }
+
+        public void Free()
+        {
+            usage = instance.UpdateHeapUse(usage, 0);
+            value = null;
+            instance = null;
+        }
+
         //private static int counter = 5;
         public static int Size(LSL_List lis)
         {
-            // VS2017 in debug mode seems to have a problem running this statement quickly:
-            //SLOW: return (!typeof(LSL_List).IsValueType && (lis == null)) ? 0 : lis.Size;
-
-            //FAST: return 33;
-            //SLOW: return (lis == null) ? 0 : 99;
-            //FAST: return ++ counter;
-
-            // VS2017 in debug mode seems content to run this quickly though:
-
             try
             {
                 return lis.Size;
@@ -141,6 +155,8 @@ namespace OpenSim.Region.ScriptEngine.Yengine
     {
         private static FieldInfo objectValueField = typeof(HeapTrackerObject).GetField("value");
         private static MethodInfo objectSaveMethod = typeof(HeapTrackerObject).GetMethod("Save");
+        private static MethodInfo objectRestoreMethod = typeof(HeapTrackerObject).GetMethod("Restore");
+        private static MethodInfo objectFreeMethod = typeof(HeapTrackerObject).GetMethod("Free");
 
         public const int HT_CHAR = 2;
         public const int HT_DELE = 8;
@@ -168,6 +184,16 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             ilGen.Emit(errorAt, OpCodes.Call, objectSaveMethod);
         }
 
+        public static void GenRestore(Token errorAt, ScriptMyILGen ilGen)
+        {
+            ilGen.Emit(errorAt, OpCodes.Call, objectRestoreMethod);
+        }
+
+        public static void GenFree(Token errorAt, ScriptMyILGen ilGen)
+        {
+            ilGen.Emit(errorAt, OpCodes.Call, objectFreeMethod);
+        }
+
         // generate CIL code to push the value on the CIL stack
         //  input:
         //   'this' pointer already pushed on CIL stack
@@ -188,6 +214,19 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             value = obj;
         }
 
+        public void Restore(object obj)
+        {
+            value = obj;
+            usage = Size(obj);
+        }
+
+        public void Free()
+        {
+            usage = instance.UpdateHeapUse(usage, 0);
+            value = null;
+            instance = null;
+        }
+
         // public so it can be used by XMRArray
         public static int Size(object obj)
         {
@@ -204,8 +243,8 @@ namespace OpenSim.Region.ScriptEngine.Yengine
                 return HT_SING;
             if(obj is int)
                 return HT_INT;
-            if(obj is LSL_Float)
+            if(obj is LSL_Float) // lsl floats are stupid doubles
-                return HT_SFLT;
+                return HT_DOUB;
             if(obj is LSL_Integer)
                 return HT_INT;
             if(obj is LSL_List)
@@ -252,7 +291,9 @@ namespace OpenSim.Region.ScriptEngine.Yengine
     public class HeapTrackerString: HeapTrackerBase
     {
         private static FieldInfo stringValueField = typeof(HeapTrackerString).GetField("value");
+        private static MethodInfo stringRestoreMethod = typeof(HeapTrackerString).GetMethod("Restore");
         private static MethodInfo stringSaveMethod = typeof(HeapTrackerString).GetMethod("Save");
+        private static MethodInfo stringFreeMethod = typeof(HeapTrackerString).GetMethod("Free");
 
         public string value;
 
@@ -271,6 +312,16 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             ilGen.Emit(errorAt, OpCodes.Call, stringSaveMethod);
         }
 
+        public static void GenRestore(Token errorAt, ScriptMyILGen ilGen)
+        {
+            ilGen.Emit(errorAt, OpCodes.Call, stringRestoreMethod);
+        }
+        
+        public static void GenFree(Token errorAt, ScriptMyILGen ilGen)
+        {
+            ilGen.Emit(errorAt, OpCodes.Call, stringFreeMethod);
+        }
+
         // generate CIL code to push the value on the CIL stack
         //  input:
         //   'this' pointer already pushed on CIL stack
@@ -291,6 +342,19 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             value = str;
         }
 
+        public void Restore(string str)
+        {
+            value = str;
+            usage = Size(str);
+        }
+
+        public void Free()
+        {
+            usage = instance.UpdateHeapUse(usage, 0);
+            value = null;
+            instance = null;
+        }
+
         public static int Size(string str)
         {
             return (str == null) ? 0 : str.Length * HeapTrackerObject.HT_CHAR;

OpenSim/Region/ScriptEngine/YEngine/XMRInstAbstract.cs

@@ -84,17 +84,36 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             heapUse = instance.UpdateHeapUse(heapUse, 0);
         }
 
+        public void Clear()
+        {
+            heapUse = 0;
+            if(iarArrays != null)
+            {
+                foreach(XMR_Array xa in iarArrays)
+                    xa.__pub_clear();
+            }
+            if(iarChars != null)
+                iarChars = new char[iarChars.Length];
+            if (iarLists != null)
+                iarLists = new LSL_List[iarLists.Length];
+            if (iarObjects != null)
+                iarObjects = new object[iarObjects.Length];
+            if(iarStrings != null)
+                iarStrings = new string[iarStrings.Length];
+        }
+
         public void AllocVarArrays(XMRInstArSizes ars)
         {
             ClearOldArrays();
-
+            int newuse = heapUse +
-            heapUse = instance.UpdateHeapUse(heapUse,
                ars.iasChars* HeapTrackerObject.HT_CHAR +
                ars.iasFloats * HeapTrackerObject.HT_SFLT +
                ars.iasIntegers * HeapTrackerObject.HT_INT +
                ars.iasRotations * HeapTrackerObject.HT_ROT +
                ars.iasVectors * HeapTrackerObject.HT_VEC +
-                ars.iasSDTIntfObjs * HeapTrackerObject.HT_DELE);
+               ars.iasSDTIntfObjs * HeapTrackerObject.HT_DELE;
+
+            heapUse = instance.UpdateHeapUse(heapUse, newuse);
 
             iarArrays = (ars.iasArrays > 0) ? new XMR_Array[ars.iasArrays] : noArrays;
             iarChars = (ars.iasChars > 0) ? new char[ars.iasChars] : noChars;
@@ -424,31 +443,13 @@ namespace OpenSim.Region.ScriptEngine.Yengine
         \**************************************************/
 
         protected int heapLimit;
-        private int heapUsed;
+        protected int heapUsed;
 
         public virtual int UpdateHeapUse(int olduse, int newuse)
         {
-            if(newuse <= olduse)
+            int newtotal = Interlocked.Add(ref heapUsed, newuse - olduse);
-                Interlocked.Add(ref heapUsed, newuse - olduse);
-            else
-            {
-                int newtotal, oldtotal;
-                do
-                {
-                    oldtotal = Interlocked.Add(ref heapUsed, 0);
-                    newtotal = oldtotal + newuse - olduse;
             if(newtotal > heapLimit)
-                    {
+                    throw new OutOfHeapException(newtotal + olduse - newuse, newtotal, heapLimit);
-                        //                        System.GC.Collect ();
-                        //                        System.GC.WaitForPendingFinalizers ();
-                        oldtotal = Interlocked.Add(ref heapUsed, 0);
-                        newtotal = oldtotal + newuse - olduse;
-                        if(newtotal > heapLimit)
-                            throw new OutOfHeapException(oldtotal, newtotal, heapLimit);
-                    }
-                } while(Interlocked.CompareExchange(ref heapUsed, newtotal, oldtotal) != oldtotal);
-            }
-
             return newuse;
         }
 

OpenSim/Region/ScriptEngine/YEngine/XMRInstMisc.cs

@@ -236,6 +236,13 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             return GetScriptFileName(m_ScriptBasePath, filename);
         }
 
+        public string GetScriptILFileName(string filename)
+        {
+            string path = Path.Combine(m_ScriptBasePath, "DebugIL");
+            Directory.CreateDirectory(path);
+            return Path.Combine(path, filename);
+        }
+
         public static string GetScriptFileName(string scriptBasePath, string filename)
         {
              // Get old path, ie, all files lumped in a single huge directory.

OpenSim/Region/ScriptEngine/YEngine/XMRInstRun.cs

@@ -841,6 +841,9 @@ namespace OpenSim.Region.ScriptEngine.Yengine
             m_SleepUntil = DateTime.MinValue;     // not doing llSleep()
             m_ResetCount++;                        // has been reset once more
 
+            heapUsed = 0;
+            glblVars.Clear();
+
              // Tell next call to 'default state_entry()' to reset all global
              // vars to their initial values.
             doGblInit = true;