Add checks monitoring framework to provide alerts if certain conditions do not hold.
Not yet in use.TeleportWork
Justin Clark-Casey (justincc)
parent
5b4b349776
commit
76bd3de2fd
|
|
@ -0,0 +1,118 @@
|
|||
/*
|
||||
* Copyright (c) Contributors, http://opensimulator.org/
|
||||
* See CONTRIBUTORS.TXT for a full list of copyright holders.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of the OpenSimulator Project nor the
|
||||
* names of its contributors may be used to endorse or promote products
|
||||
* derived from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Text;
|
||||
|
||||
namespace OpenSim.Framework.Monitoring
|
||||
{
|
||||
public class Check
|
||||
{
|
||||
// private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
public static readonly char[] DisallowedShortNameCharacters = { '.' };
|
||||
|
||||
/// <summary>
|
||||
/// Category of this stat (e.g. cache, scene, etc).
|
||||
/// </summary>
|
||||
public string Category { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Containing name for this stat.
|
||||
/// FIXME: In the case of a scene, this is currently the scene name (though this leaves
|
||||
/// us with a to-be-resolved problem of non-unique region names).
|
||||
/// </summary>
|
||||
/// <value>
|
||||
/// The container.
|
||||
/// </value>
|
||||
public string Container { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Action used to check whether alert should go off.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Should return true if check passes. False otherwise.
|
||||
/// </remarks>
|
||||
public Func<Check, bool> CheckFunc { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Message from the last failure, if any. If there is no message or no failure then will be null.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Should be set by the CheckFunc when applicable.
|
||||
/// </remarks>
|
||||
public string LastFailureMessage { get; set; }
|
||||
|
||||
public StatVerbosity Verbosity { get; private set; }
|
||||
public string ShortName { get; private set; }
|
||||
public string Name { get; private set; }
|
||||
public string Description { get; private set; }
|
||||
|
||||
public Check(
|
||||
string shortName,
|
||||
string name,
|
||||
string description,
|
||||
string category,
|
||||
string container,
|
||||
Func<Check, bool> checkFunc,
|
||||
StatVerbosity verbosity)
|
||||
{
|
||||
if (ChecksManager.SubCommands.Contains(category))
|
||||
throw new Exception(
|
||||
string.Format("Alert cannot be in category '{0}' since this is reserved for a subcommand", category));
|
||||
|
||||
foreach (char c in DisallowedShortNameCharacters)
|
||||
{
|
||||
if (shortName.IndexOf(c) != -1)
|
||||
throw new Exception(string.Format("Alert name {0} cannot contain character {1}", shortName, c));
|
||||
}
|
||||
|
||||
ShortName = shortName;
|
||||
Name = name;
|
||||
Description = description;
|
||||
Category = category;
|
||||
Container = container;
|
||||
CheckFunc = checkFunc;
|
||||
Verbosity = verbosity;
|
||||
}
|
||||
|
||||
public bool CheckIt()
|
||||
{
|
||||
return CheckFunc(this);
|
||||
}
|
||||
|
||||
public virtual string ToConsoleString()
|
||||
{
|
||||
return string.Format(
|
||||
"{0}.{1}.{2} - {3}",
|
||||
Category,
|
||||
Container,
|
||||
ShortName,
|
||||
Description);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
/*
|
||||
* Copyright (c) Contributors, http://opensimulator.org/
|
||||
* See CONTRIBUTORS.TXT for a full list of copyright holders.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of the OpenSimulator Project nor the
|
||||
* names of its contributors may be used to endorse or promote products
|
||||
* derived from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using log4net;
|
||||
|
||||
namespace OpenSim.Framework.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Static class used to register/deregister checks on runtime conditions.
|
||||
/// </summary>
|
||||
public static class ChecksManager
|
||||
{
|
||||
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
|
||||
|
||||
// Subcommand used to list other stats.
|
||||
public const string ListSubCommand = "list";
|
||||
|
||||
// All subcommands
|
||||
public static HashSet<string> SubCommands = new HashSet<string> { ListSubCommand };
|
||||
|
||||
/// <summary>
|
||||
/// Checks categorized by category/container/shortname
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Do not add or remove directly from this dictionary.
|
||||
/// </remarks>
|
||||
public static SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Check>>> RegisteredChecks
|
||||
= new SortedDictionary<string, SortedDictionary<string, SortedDictionary<string, Check>>>();
|
||||
|
||||
public static void RegisterConsoleCommands(ICommandConsole console)
|
||||
{
|
||||
console.Commands.AddCommand(
|
||||
"General",
|
||||
false,
|
||||
"show checks",
|
||||
"show checks",
|
||||
"Show checks configured for this server",
|
||||
"If no argument is specified then info on all checks will be shown.\n"
|
||||
+ "'list' argument will show check categories.\n"
|
||||
+ "THIS FACILITY IS EXPERIMENTAL",
|
||||
HandleShowchecksCommand);
|
||||
}
|
||||
|
||||
public static void HandleShowchecksCommand(string module, string[] cmd)
|
||||
{
|
||||
ICommandConsole con = MainConsole.Instance;
|
||||
|
||||
if (cmd.Length > 2)
|
||||
{
|
||||
foreach (string name in cmd.Skip(2))
|
||||
{
|
||||
string[] components = name.Split('.');
|
||||
|
||||
string categoryName = components[0];
|
||||
// string containerName = components.Length > 1 ? components[1] : null;
|
||||
|
||||
if (categoryName == ListSubCommand)
|
||||
{
|
||||
con.Output("check categories available are:");
|
||||
|
||||
foreach (string category in RegisteredChecks.Keys)
|
||||
con.OutputFormat(" {0}", category);
|
||||
}
|
||||
// else
|
||||
// {
|
||||
// SortedDictionary<string, SortedDictionary<string, Check>> category;
|
||||
// if (!Registeredchecks.TryGetValue(categoryName, out category))
|
||||
// {
|
||||
// con.OutputFormat("No such category as {0}", categoryName);
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// if (String.IsNullOrEmpty(containerName))
|
||||
// {
|
||||
// OutputConfiguredToConsole(con, category);
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// SortedDictionary<string, Check> container;
|
||||
// if (category.TryGetValue(containerName, out container))
|
||||
// {
|
||||
// OutputContainerChecksToConsole(con, container);
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// con.OutputFormat("No such container {0} in category {1}", containerName, categoryName);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
OutputAllChecksToConsole(con);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a statistic.
|
||||
/// </summary>
|
||||
/// <param name='stat'></param>
|
||||
/// <returns></returns>
|
||||
public static bool RegisterCheck(Check check)
|
||||
{
|
||||
SortedDictionary<string, SortedDictionary<string, Check>> category = null, newCategory;
|
||||
SortedDictionary<string, Check> container = null, newContainer;
|
||||
|
||||
lock (RegisteredChecks)
|
||||
{
|
||||
// Check name is not unique across category/container/shortname key.
|
||||
// XXX: For now just return false. This is to avoid problems in regression tests where all tests
|
||||
// in a class are run in the same instance of the VM.
|
||||
if (TryGetCheckParents(check, out category, out container))
|
||||
return false;
|
||||
|
||||
// We take a copy-on-write approach here of replacing dictionaries when keys are added or removed.
|
||||
// This means that we don't need to lock or copy them on iteration, which will be a much more
|
||||
// common operation after startup.
|
||||
if (container != null)
|
||||
newContainer = new SortedDictionary<string, Check>(container);
|
||||
else
|
||||
newContainer = new SortedDictionary<string, Check>();
|
||||
|
||||
if (category != null)
|
||||
newCategory = new SortedDictionary<string, SortedDictionary<string, Check>>(category);
|
||||
else
|
||||
newCategory = new SortedDictionary<string, SortedDictionary<string, Check>>();
|
||||
|
||||
newContainer[check.ShortName] = check;
|
||||
newCategory[check.Container] = newContainer;
|
||||
RegisteredChecks[check.Category] = newCategory;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deregister an check
|
||||
/// </summary>>
|
||||
/// <param name='stat'></param>
|
||||
/// <returns></returns>
|
||||
public static bool DeregisterCheck(Check check)
|
||||
{
|
||||
SortedDictionary<string, SortedDictionary<string, Check>> category = null, newCategory;
|
||||
SortedDictionary<string, Check> container = null, newContainer;
|
||||
|
||||
lock (RegisteredChecks)
|
||||
{
|
||||
if (!TryGetCheckParents(check, out category, out container))
|
||||
return false;
|
||||
|
||||
newContainer = new SortedDictionary<string, Check>(container);
|
||||
newContainer.Remove(check.ShortName);
|
||||
|
||||
newCategory = new SortedDictionary<string, SortedDictionary<string, Check>>(category);
|
||||
newCategory.Remove(check.Container);
|
||||
|
||||
newCategory[check.Container] = newContainer;
|
||||
RegisteredChecks[check.Category] = newCategory;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public static bool TryGetCheckParents(
|
||||
Check check,
|
||||
out SortedDictionary<string, SortedDictionary<string, Check>> category,
|
||||
out SortedDictionary<string, Check> container)
|
||||
{
|
||||
category = null;
|
||||
container = null;
|
||||
|
||||
lock (RegisteredChecks)
|
||||
{
|
||||
if (RegisteredChecks.TryGetValue(check.Category, out category))
|
||||
{
|
||||
if (category.TryGetValue(check.Container, out container))
|
||||
{
|
||||
if (container.ContainsKey(check.ShortName))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static void CheckChecks()
|
||||
{
|
||||
lock (RegisteredChecks)
|
||||
{
|
||||
foreach (SortedDictionary<string, SortedDictionary<string, Check>> category in RegisteredChecks.Values)
|
||||
{
|
||||
foreach (SortedDictionary<string, Check> container in category.Values)
|
||||
{
|
||||
foreach (Check check in container.Values)
|
||||
{
|
||||
if (!check.CheckIt())
|
||||
m_log.WarnFormat(
|
||||
"[CHECKS MANAGER]: Check {0}.{1}.{2} failed with message {3}", check.Category, check.Container, check.ShortName, check.LastFailureMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void OutputAllChecksToConsole(ICommandConsole con)
|
||||
{
|
||||
foreach (var category in RegisteredChecks.Values)
|
||||
{
|
||||
OutputCategoryChecksToConsole(con, category);
|
||||
}
|
||||
}
|
||||
|
||||
private static void OutputCategoryChecksToConsole(
|
||||
ICommandConsole con, SortedDictionary<string, SortedDictionary<string, Check>> category)
|
||||
{
|
||||
foreach (var container in category.Values)
|
||||
{
|
||||
OutputContainerChecksToConsole(con, container);
|
||||
}
|
||||
}
|
||||
|
||||
private static void OutputContainerChecksToConsole(ICommandConsole con, SortedDictionary<string, Check> container)
|
||||
{
|
||||
foreach (Check check in container.Values)
|
||||
{
|
||||
con.Output(check.ToConsoleString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -35,9 +35,9 @@ using OpenMetaverse.StructuredData;
|
|||
namespace OpenSim.Framework.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Singleton used to provide access to statistics reporters
|
||||
/// Static class used to register/deregister/fetch statistics
|
||||
/// </summary>
|
||||
public class StatsManager
|
||||
public static class StatsManager
|
||||
{
|
||||
// Subcommand used to list other stats.
|
||||
public const string AllSubCommand = "all";
|
||||
|
|
@ -257,7 +257,7 @@ namespace OpenSim.Framework.Monitoring
|
|||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Registers a statistic.
|
||||
/// Register a statistic.
|
||||
/// </summary>
|
||||
/// <param name='stat'></param>
|
||||
/// <returns></returns>
|
||||
|
|
|
|||
|
|
@ -380,6 +380,7 @@ namespace OpenSim.Framework.Monitoring
|
|||
if (MemoryWatchdog.Enabled)
|
||||
MemoryWatchdog.Update();
|
||||
|
||||
ChecksManager.CheckChecks();
|
||||
StatsManager.RecordStats();
|
||||
|
||||
m_watchdogTimer.Start();
|
||||
|
|
|
|||
|
|
@ -272,6 +272,7 @@ namespace OpenSim.Framework.Servers
|
|||
"shutdown",
|
||||
"Quit the application", (mod, args) => Shutdown());
|
||||
|
||||
ChecksManager.RegisterConsoleCommands(m_console);
|
||||
StatsManager.RegisterConsoleCommands(m_console);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue