Port the safer database access and the cure for "Chicken Bones" which

are artefacts that can be created by fsck and would make standard
FSAssets choke. These refinements were tested on OSGrid.

OpenSim/Data/MySQL/MySQLFSAssetData.cs

@@ -41,10 +41,8 @@ namespace OpenSim.Data.MySQL
     {
         private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
 
-        protected MySqlConnection m_Connection = null;
         protected string m_ConnectionString;
         protected string m_Table;
-        protected Object m_connLock = new Object();
 
         /// <summary>
         /// Number of days that must pass before we update the access time on an asset when it has been fetched
@@ -75,11 +73,12 @@ namespace OpenSim.Data.MySQL
 
             try
             {
-                OpenDatabase();
+                using (MySqlConnection conn = new MySqlConnection(m_ConnectionString))
-
+                {
-                Migration m = new Migration(m_Connection, Assembly, "FSAssetStore");
+                    Migration m = new Migration(conn, Assembly, "FSAssetStore");
                     m.Update();
                 }
+            }
             catch (MySqlException e)
             {
                 m_log.ErrorFormat("[FSASSETS]: Can't connect to database: {0}", e.Message.ToString());
@@ -100,70 +99,33 @@ namespace OpenSim.Data.MySQL
 
         #endregion
 
-        private bool OpenDatabase()
+        private bool ExecuteNonQuery(MySqlCommand cmd)
+        {
+            using (MySqlConnection conn = new MySqlConnection(m_ConnectionString))
             {
                 try
                 {
-                m_Connection = new MySqlConnection(m_ConnectionString);
+                    conn.Open();
-
-                m_Connection.Open();
                 }
                 catch (MySqlException e)
                 {
-                m_log.ErrorFormat("[FSASSETS]: Can't connect to database: {0}",
+                    m_log.ErrorFormat("[FSASSETS]: Database open failed with {0}", e.ToString());
-                        e.Message.ToString());
-
                     return false;
                 }
 
-            return true;
+                cmd.Connection = conn;
-        }
-
-        private IDataReader ExecuteReader(MySqlCommand c)
-        {
-            IDataReader r = null;
-            MySqlConnection connection = (MySqlConnection) ((ICloneable)m_Connection).Clone();
-            connection.Open();
-            c.Connection = connection;
-
-            r = c.ExecuteReader();
-
-            return r;
-        }
-
-        private void ExecuteNonQuery(MySqlCommand c)
-        {
-            lock (m_connLock)
-            {
-                bool errorSeen = false;
-
-                while (true)
-                {
                 try
                 {
-                        c.ExecuteNonQuery();
+                    cmd.ExecuteNonQuery();
                 }
-                    catch (MySqlException)
+                catch (MySqlException e)
                 {
-                        System.Threading.Thread.Sleep(500);
+                    m_log.ErrorFormat("[FSASSETS]: Query {0} failed with {1}", cmd.CommandText, e.ToString());
-
+                    return false;
-                        m_Connection.Close();
-                        m_Connection = (MySqlConnection) ((ICloneable)m_Connection).Clone();
-                        m_Connection.Open();
-                        c.Connection = m_Connection;
-
-                        if (!errorSeen)
-                        {
-                            errorSeen = true;
-                            continue;
                 }
-                        m_log.ErrorFormat("[FSASSETS] MySQL command: {0}", c.CommandText);
-                        throw;
             }
 
-                    break;
+            return true;
-                }
-            }
         }
 
         #region IFSAssetDataPlugin Members
@@ -172,21 +134,29 @@ namespace OpenSim.Data.MySQL
         {
             hash = String.Empty;
 
-            MySqlCommand cmd = new MySqlCommand();
+            AssetMetadata meta = new AssetMetadata();
 
-            cmd.CommandText = String.Format("select id, name, description, type, hash, create_time, access_time, asset_flags from {0} where id = ?id", m_Table);
+            using (MySqlConnection conn = new MySqlConnection(m_ConnectionString))
-            cmd.Parameters.AddWithValue("?id", id);
-
-            IDataReader reader = ExecuteReader(cmd);
-
-            if (!reader.Read())
             {
-                reader.Close();
+                try
-                FreeCommand(cmd);
+                {
+                    conn.Open();
+                }
+                catch (MySqlException e)
+                {
+                    m_log.ErrorFormat("[FSASSETS]: Database open failed with {0}", e.ToString());
                     return null;
                 }
 
-            AssetMetadata meta = new AssetMetadata();
+                using (MySqlCommand cmd = conn.CreateCommand())
+                {
+                    cmd.CommandText = String.Format("select id, name, description, type, hash, create_time, asset_flags from {0} where id = ?id", m_Table);
+                    cmd.Parameters.AddWithValue("?id", id);
+
+                    using (IDataReader reader = cmd.ExecuteReader())
+                    {
+                        if (!reader.Read())
+                            return null;
 
                         hash = reader["hash"].ToString();
 
@@ -201,34 +171,41 @@ namespace OpenSim.Data.MySQL
                         meta.Flags = (AssetFlags)Convert.ToInt32(reader["asset_flags"]);
 
                         int AccessTime = Convert.ToInt32(reader["access_time"]);
+                        UpdateAccessTime(AccessTime);
+                    }
+                }
 
-            reader.Close();
+            }
-
-            UpdateAccessTime(AccessTime, cmd);
-
-            FreeCommand(cmd);
 
             return meta;
         }
 
-        private void UpdateAccessTime(int AccessTime, MySqlCommand cmd)
+        private void UpdateAccessTime(int AccessTime)
         {
             // Reduce DB work by only updating access time if asset hasn't recently been accessed
             // 0 By Default, Config option is "DaysBetweenAccessTimeUpdates"
             if (DaysBetweenAccessTimeUpdates > 0 && (DateTime.UtcNow - Utils.UnixTimeToDateTime(AccessTime)).TotalDays < DaysBetweenAccessTimeUpdates)
                 return;
 
+            using (MySqlConnection conn = new MySqlConnection(m_ConnectionString))
+            {
+                try
+                {
+                    conn.Open();
+                }
+                catch (MySqlException e)
+                {
+                    m_log.ErrorFormat("[FSASSETS]: Database open failed with {0}", e.ToString());
+                    return;
+                }
+
+                using (MySqlCommand cmd = conn.CreateCommand())
+                {
                     cmd.CommandText = String.Format("UPDATE {0} SET `access_time` = UNIX_TIMESTAMP() WHERE `id` = ?id", m_Table);
 
                     cmd.ExecuteNonQuery();
                 }
-
+            }
-        protected void FreeCommand(MySqlCommand cmd)
-        {
-            MySqlConnection c = cmd.Connection;
-            cmd.Dispose();
-            c.Close();
-            c.Dispose();
         }
 
         public bool Store(AssetMetadata meta, string hash)
@@ -238,8 +215,8 @@ namespace OpenSim.Data.MySQL
                 string oldhash;
                 AssetMetadata existingAsset = Get(meta.ID, out oldhash);
 
-                MySqlCommand cmd = m_Connection.CreateCommand();
+                using (MySqlCommand cmd = new MySqlCommand())
-
+                {
                     cmd.Parameters.AddWithValue("?id", meta.ID);
                     cmd.Parameters.AddWithValue("?name", meta.Name);
                     cmd.Parameters.AddWithValue("?description", meta.Description);
@@ -253,8 +230,6 @@ namespace OpenSim.Data.MySQL
 
                         ExecuteNonQuery(cmd);
 
-                    cmd.Dispose();
-
                         return true;
                     }
 
@@ -262,7 +237,7 @@ namespace OpenSim.Data.MySQL
 
                     //ExecuteNonQuery(cmd);
 
-                cmd.Dispose();
+                }
                 return false;
             }
             catch(Exception e)
@@ -283,12 +258,28 @@ namespace OpenSim.Data.MySQL
             if (uuids.Length == 0)
                 return new bool[0];
 
+            bool[] results = new bool[uuids.Length];
+            for (int i = 0; i < uuids.Length; i++)
+                results[i] = false;
+
             HashSet<UUID> exists = new HashSet<UUID>();
 
             string ids = "'" + string.Join("','", uuids) + "'";
             string sql = string.Format("select id from {1} where id in ({0})", ids, m_Table);
 
-            using (MySqlCommand cmd = m_Connection.CreateCommand())
+            using (MySqlConnection conn = new MySqlConnection(m_ConnectionString))
+            {
+                try
+                {
+                    conn.Open();
+                }
+                catch (MySqlException e)
+                {
+                    m_log.ErrorFormat("[FSASSETS]: Failed to open database: {0}", e.ToString());
+                    return results;
+                }
+
+                using (MySqlCommand cmd = conn.CreateCommand())
                 {
                     cmd.CommandText = sql;
 
@@ -301,8 +292,8 @@ namespace OpenSim.Data.MySQL
                         }
                     }
                 }
+            }
 
-            bool[] results = new bool[uuids.Length];
             for (int i = 0; i < uuids.Length; i++)
                 results[i] = exists.Contains(uuids[i]);
             return results;
@@ -310,44 +301,58 @@ namespace OpenSim.Data.MySQL
 
         public int Count()
         {
-            MySqlCommand cmd = m_Connection.CreateCommand();
+            int count = 0;
+
+            using (MySqlConnection conn = new MySqlConnection(m_ConnectionString))
+            {
+                try
+                {
+                    conn.Open();
+                }
+                catch (MySqlException e)
+                {
+                    m_log.ErrorFormat("[FSASSETS]: Failed to open database: {0}", e.ToString());
+                    return 0;
+                }
+
+                MySqlCommand cmd = conn.CreateCommand();
 
                 cmd.CommandText = String.Format("select count(*) as count from {0}", m_Table);
 
-            IDataReader reader = ExecuteReader(cmd);
+                using (IDataReader reader = cmd.ExecuteReader())
-
+                {
                     reader.Read();
 
-            int count = Convert.ToInt32(reader["count"]);
+                    count = Convert.ToInt32(reader["count"]);
-
+                }
-            reader.Close();
+            }
-            FreeCommand(cmd);
 
             return count;
         }
 
         public bool Delete(string id)
         {
-            using (MySqlCommand cmd = m_Connection.CreateCommand())
+            MySqlCommand cmd = new MySqlCommand();
-            {
+
             cmd.CommandText = String.Format("delete from {0} where id = ?id", m_Table);
 
             cmd.Parameters.AddWithValue("?id", id);
 
             ExecuteNonQuery(cmd);
-            }
+
+            cmd.Dispose();
 
             return true;
         }
 
         public void Import(string conn, string table, int start, int count, bool force, FSStoreDelegate store)
         {
-            MySqlConnection importConn;
+            int imported = 0;
 
+            using (MySqlConnection importConn = new MySqlConnection(conn))
+            {
                 try
                 {
-                importConn = new MySqlConnection(conn);
-
                     importConn.Open();
                 }
                 catch (MySqlException e)
@@ -358,10 +363,8 @@ namespace OpenSim.Data.MySQL
                     return;
                 }
 
-            int imported = 0;
+                using (MySqlCommand cmd = importConn.CreateCommand())
-
+                {
-            MySqlCommand cmd = importConn.CreateCommand();
-
                     string limit = String.Empty;
                     if (count != -1)
                     {
@@ -371,8 +374,8 @@ namespace OpenSim.Data.MySQL
                     cmd.CommandText = String.Format("select * from {0}{1}", table, limit);
 
                     MainConsole.Instance.Output("Querying database");
-            IDataReader reader = cmd.ExecuteReader();
+                    using (IDataReader reader = cmd.ExecuteReader())
-
+                    {
                         MainConsole.Instance.Output("Reading data");
 
                         while (reader.Read())
@@ -401,10 +404,10 @@ namespace OpenSim.Data.MySQL
 
                             imported++;
                         }
+                    }
 
-            reader.Close();
+                }
-            cmd.Dispose();
+            }
-            importConn.Close();
 
             MainConsole.Instance.Output(String.Format("Import done, {0} assets imported", imported));
         }

OpenSim/Services/FSAssetService/FSAssetService.cs

@@ -119,8 +119,8 @@ namespace OpenSim.Services.FSAssetService
 
             // Get Database Connector from Asset Config (If present)
             string dllName = assetConfig.GetString("StorageProvider", string.Empty);
-            string m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty);
+            string connectionString = assetConfig.GetString("ConnectionString", string.Empty);
-            string m_Realm = assetConfig.GetString("Realm", "fsassets");
+            string realm = assetConfig.GetString("Realm", "fsassets");
 
             int SkipAccessTimeDays = assetConfig.GetInt("DaysBetweenAccessTimeUpdates", 0);
 
@@ -132,15 +132,15 @@ namespace OpenSim.Services.FSAssetService
                 if (dllName == String.Empty)
                     dllName = dbConfig.GetString("StorageProvider", String.Empty);
                 
-                if (m_ConnectionString == String.Empty)
+                if (connectionString == String.Empty)
-                    m_ConnectionString = dbConfig.GetString("ConnectionString", String.Empty);
+                    connectionString = dbConfig.GetString("ConnectionString", String.Empty);
             }
 
             // No databse connection found in either config
             if (dllName.Equals(String.Empty))
                 throw new Exception("No StorageProvider configured");
 
-            if (m_ConnectionString.Equals(String.Empty))
+            if (connectionString.Equals(String.Empty))
                 throw new Exception("Missing database connection string");
 
             // Create Storage Provider
@@ -150,7 +150,7 @@ namespace OpenSim.Services.FSAssetService
                 throw new Exception(string.Format("Could not find a storage interface in the module {0}", dllName));
 
             // Initialize DB And perform any migrations required
-            m_DataConnector.Initialise(m_ConnectionString, m_Realm, SkipAccessTimeDays);
+            m_DataConnector.Initialise(connectionString, realm, SkipAccessTimeDays);
 
             // Setup Fallback Service
             string str = assetConfig.GetString("FallbackService", string.Empty);
@@ -232,7 +232,7 @@ namespace OpenSim.Services.FSAssetService
 
         private void Writer()
         {
-            m_log.Info("[FSASSETS]: Writer started");
+            m_log.Info("[ASSET]: Writer started");
 
             while (true)
             {
@@ -246,8 +246,71 @@ namespace OpenSim.Services.FSAssetService
                         string hash = Path.GetFileNameWithoutExtension(files[i]);
                         string s = HashToFile(hash);
                         string diskFile = Path.Combine(m_FSBase, s);
+                        bool pathOk = false;
 
+                        // The cure for chicken bones!
+                        while(true)
+                        {
+                            try
+                            {
+                                // Try to make the directory we need for this file
                                 Directory.CreateDirectory(Path.GetDirectoryName(diskFile));
+                                pathOk = true;
+                                break;
+                            }
+                            catch (System.IO.IOException)
+                            {
+                                // Creating the directory failed. This can't happen unless
+                                // a part of the path already exists as a file. Sadly the
+                                // SRAS data contains such files.
+                                string d = Path.GetDirectoryName(diskFile);
+
+                                // Test each path component in turn. If we can successfully
+                                // make a directory, the level below must be the chicken bone.
+                                while (d.Length > 0)
+                                {
+                                    Console.WriteLine(d);
+                                    try
+                                    {
+                                        Directory.CreateDirectory(Path.GetDirectoryName(d));
+                                    }
+                                    catch (System.IO.IOException)
+                                    {
+                                        d = Path.GetDirectoryName(d);
+
+                                        // We failed making the directory and need to
+                                        // go up a bit more
+                                        continue;
+                                    }
+
+                                    // We succeeded in making the directory and (d) is
+                                    // the chicken bone
+                                    break;
+                                }
+
+                                // Is the chicken alive?
+                                if (d.Length > 0)
+                                {
+                                    Console.WriteLine(d);
+
+                                    FileAttributes attr = File.GetAttributes(d);
+
+                                    if ((attr & FileAttributes.Directory) == 0)
+                                    {
+                                        // The chicken bone should be resolved.
+                                        // Return to writing the file.
+                                        File.Delete(d);
+                                        continue;
+                                    }
+                                }
+                            }
+                            // Could not resolve, skipping
+                            m_log.ErrorFormat("[ASSET]: Could not resolve path creation error for {0}", diskFile);
+                            break;
+                        }
+
+                        if (pathOk)
+                        {
                             try
                             {
                                 byte[] data = File.ReadAllBytes(files[i]);
@@ -269,10 +332,12 @@ namespace OpenSim.Services.FSAssetService
                                     throw;
                             }
                         }
+                    }
+
                     int totalTicks = System.Environment.TickCount - tickCount;
                     if (totalTicks > 0) // Wrap?
                     {
-                        m_log.InfoFormat("[FSASSETS]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length);
+                        m_log.InfoFormat("[ASSET]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length);
                     }
                 }
 
@@ -292,20 +357,25 @@ namespace OpenSim.Services.FSAssetService
             if (hash == null || hash.Length < 10)
                 return "junkyard";
 
+            /*
+             * The code below is the OSGrid code.
+             * This should probably become a config option.
+             */
+            /*
             return Path.Combine(hash.Substring(0, 3),
                    Path.Combine(hash.Substring(3, 3)));
+            */
+
             /*
              * The below is what core would normally use.
              * This is modified to work in OSGrid, as seen
              * above, because the SRAS data is structured
              * that way.
              */
-            /*
             return Path.Combine(hash.Substring(0, 2),
                    Path.Combine(hash.Substring(2, 2),
                    Path.Combine(hash.Substring(4, 2),
                    hash.Substring(6, 4))));
-            */
         }
 
         private bool AssetExists(string hash)