Skip to content

Commit

Permalink
IoT Samrt Grig v2 - 1.4M rows/sec
Browse files Browse the repository at this point in the history
  • Loading branch information
@perrysk-msft
perrysk-msft committed Nov 7, 2017
1 parent e0268de commit aaa0465
Show file tree
Hide file tree
Showing 19 changed files with 614 additions and 489 deletions.
31 changes: 18 additions & 13 deletions samples/applications/iot-smart-grid/App.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,26 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
</startup>
<connectionStrings>
<!--<add name="Db" connectionString="Server=tcp:<server>.database.windows.net,1433;Database=PowerConsumption;User ID=<user>;Password=<password>;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"/>-->
<clear />
<!--<add name="Db" connectionString="Server=tcp:(YOUR_SQLSERVERNAME).database.windows.net,1433;Database=(YOUR_DATABASE_NAME);User ID=(YOUR_USERNAME);Password=(YOUR_PASSWORD);Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"/>-->
<add name="Db" connectionString="Server=.;Database=PowerConsumption;Integrated Security=True"/>
</connectionStrings>
<appSettings>
<add key="insertSPName" value="InsertMeterMeasurement"/> <!--Stored Procedure Name-->
<add key="numberOfTasks" value="50"/> <!--Number of concurrent async tasks that the Data Generator will use-->
<add key="numberOfMeters" value="1000000"/> <!--Number of unique meter Ids-->
<add key="batchSize" value="1000"/> <!--Row Batch Size that every task produces-->
<add key="commandDelay" value="1500"/> <!--Delay between sql commands. You can set this to 0 for max high volume workload-->
<add key="commandTimeout" value="600"/> <!--SQL Command Timeout-->
<add key="enableShock" value="1"/> <!--Flag that turns on/off the data shock. This should be set to 0 for max high volume workload-->
<add key="shockFrequency" value="35000"/> <!--The Frequency that the code sets the commandDelay to 0-->
<add key="shockDuration" value="4000"/> <!--The Duration that the code keeps the commandDelay to 0-->
<add key="rpsFrequency" value="2000"/> <!--How frequently the Data Generator Rows Per Second(RPS) is polled-->
<add key="logFileName" value="log.txt"/> <!--Log File Path-->
<add key="powerBIDesktopPath" value="C:\Program Files\Microsoft Power BI Desktop\bin\PBIDesktop.exe"/><!--PowerBIDesktop.exe path-->
<add key="insertSPName" value="InsertMeterMeasurement"/> <!--Stored Procedure Name-->
<add key="numberOfDataLoadTasks" value="70"/> <!--Number of concurrent async tasks that the Data Generator will use (per SQL Connection)-->
<add key="dataLoadCommandDelay" value="2000"/> <!--Delay between sql commands. You can set this to 0 for max high volume workload-->
<add key="batchSize" value="70000"/> <!--Insert: Row Batch Size that every task produces-->
<add key="deleteSPName" value="InsertMeterMeasurementHistory"/> <!--Stored Procedure Name-->
<add key="numberOfOffLoadTasks" value="50"/> <!--Number of concurrent async tasks that the Data Generator will use (per SQL Connection)-->
<add key="offLoadCommandDelay" value="0"/> <!--Delay between sql commands. You can set this to 0 for max high volume workload-->
<add key="deleteBatchSize" value="1000000"/> <!--Delete: Row Batch Size that every task produces-->
<add key="numberOfMeters" value="75000000"/> <!--Number of unique meters-->
<add key="commandTimeout" value="600"/> <!--SQL Command Timeout-->
<add key="rpsFrequency" value="500"/> <!--How frequently the Data Generator Rows Per Second(RPS) is polled-->
<add key="logFileName" value="log.txt"/> <!--Log File Path-->
<add key="delayStart" value="0"/> <!--Delay Graph Interval-->
<add key="appRunDuration" value="1800000"/> <!--Run App Time: 30 mins-->
<add key="numberOfRowsOfloadLimit" value="1400000"/> <!--Number Of Rows Ofload Limit-->
<add key="powerBIDesktopPath" value="C:\Program Files\Microsoft Power BI Desktop\bin\PBIDesktop.exe"/> <!--PowerBIDesktop.exe path-->
</appSettings>
</configuration>
84 changes: 36 additions & 48 deletions samples/applications/iot-smart-grid/ConsoleClient/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,40 +36,39 @@ This code sample simulates an IoT Smart Grid scenario where multiple IoT power m
shock absorber scenario: https://blogs.technet.microsoft.com/dataplatforminsider/2013/09/19/in-memory-oltp-common-design-pattern-high-data-input-rateshock-absorber/.
Every async task in the Data Generator produces a batch of records with random values in order to simulate the data of an IoT power meter.
It then calls a natively compiled stored procedure, that accepts an memory optimized table valued parameter (TVP), to insert the data into an memory optimized SQL Server table.
In addition to the in-memory features, the sample is leveraging System-Versioned Temporal Tables: https://msdn.microsoft.com/en-us/library/dn935015.aspx for building version history,
Clustered Columnstore Index: https://msdn.microsoft.com/en-us/library/dn817827.aspx) for enabling real time operational analytics, and
In addition to the in-memory features, the sample is offloading historical values to a Clustered Columnstore Index: https://msdn.microsoft.com/en-us/library/dn817827.aspx) for enabling real time operational analytics, and
Power BI: https://powerbi.microsoft.com/en-us/desktop/ for data visualization.
*/
namespace ConsoleClient
{
class Program
{
static SqlDataGenerator dataGenerator;
static string connection;
static string[] connection;
static string spName;
static string logFileName;
static string powerBIDesktopPath;
static int tasks;
static int meters;
static int batchSize;
static int delay;
static int commandTimeout;
static int shockFrequency;
static int shockDuration;
static int rpsFrequency;
static int enableShock;
static Timer mainTimer = new Timer();
static int rpsFrequency;
static int numberOfDataLoadTasks;
static int numberOfOffLoadTasks;
static int deleteBatchSize;
static string deleteSPName;
static int dataLoadCommandDelay;
static int offLoadCommandDelay;
static int delayStart;
static int appRunDuration;
static int numberOfRowsOfloadLimit;
static Timer rpsTimer = new Timer();
static Timer shockTimer = new Timer();

static void Main(string[] args)
{
Init();
dataGenerator = new SqlDataGenerator(connection, spName, commandTimeout, meters, tasks, delay, batchSize, ExceptionCallback);

mainTimer.Elapsed += mainTimer_Tick;
dataGenerator = new SqlDataGenerator(connection, spName, commandTimeout, meters, numberOfDataLoadTasks, dataLoadCommandDelay, batchSize, deleteSPName, numberOfOffLoadTasks, offLoadCommandDelay, deleteBatchSize, numberOfRowsOfloadLimit, ExceptionCallback);
rpsTimer.Elapsed += rpsTimer_Tick;
shockTimer.Elapsed += shockTimer_Tick;


string commandString = string.Empty;
Console.ForegroundColor = ConsoleColor.White;
Expand Down Expand Up @@ -129,9 +128,7 @@ static async void Start()
{
if (!dataGenerator.IsRunning)
{
if(enableShock == 1) mainTimer.Start();
rpsTimer.Start();

await dataGenerator.RunAsync();
}
}
Expand All @@ -144,10 +141,7 @@ static async void Stop()
{
if (dataGenerator.IsRunning)
{
if (enableShock == 1) mainTimer.Stop();
rpsTimer.Stop();
if (enableShock == 1) shockTimer.Stop();

await dataGenerator.StopAsync();
dataGenerator.RpsReset();
}
Expand Down Expand Up @@ -177,47 +171,47 @@ static void Init()
{
try
{
int numberOfSqlConnections = ConfigurationManager.ConnectionStrings.Count;
connection = new string[numberOfSqlConnections];
// Read Config Settings
connection = ConfigurationManager.ConnectionStrings["Db"].ConnectionString;
for (int i = 0; i < numberOfSqlConnections; i++)
{
connection[i] = ConfigurationManager.ConnectionStrings[i].ConnectionString;
}

spName = ConfigurationManager.AppSettings["insertSPName"];
logFileName = ConfigurationManager.AppSettings["logFileName"];
powerBIDesktopPath = ConfigurationManager.AppSettings["powerBIDesktopPath"];
tasks = int.Parse(ConfigurationManager.AppSettings["numberOfTasks"]);
meters = int.Parse(ConfigurationManager.AppSettings["numberOfMeters"]);
numberOfDataLoadTasks = int.Parse(ConfigurationManager.AppSettings["numberOfDataLoadTasks"]);
dataLoadCommandDelay = int.Parse(ConfigurationManager.AppSettings["dataLoadCommandDelay"]);
batchSize = int.Parse(ConfigurationManager.AppSettings["batchSize"]);
delay = int.Parse(ConfigurationManager.AppSettings["commandDelay"]);
deleteSPName = ConfigurationManager.AppSettings["deleteSPName"];
numberOfOffLoadTasks = int.Parse(ConfigurationManager.AppSettings["numberOfOffLoadTasks"]);
offLoadCommandDelay = int.Parse(ConfigurationManager.AppSettings["offLoadCommandDelay"]);
deleteBatchSize = int.Parse(ConfigurationManager.AppSettings["deleteBatchSize"]);
meters = int.Parse(ConfigurationManager.AppSettings["numberOfMeters"]);
commandTimeout = int.Parse(ConfigurationManager.AppSettings["commandTimeout"]);
shockFrequency = int.Parse(ConfigurationManager.AppSettings["shockFrequency"]);
shockDuration = int.Parse(ConfigurationManager.AppSettings["shockDuration"]);
enableShock = int.Parse(ConfigurationManager.AppSettings["enableShock"]);

delayStart = int.Parse(ConfigurationManager.AppSettings["delayStart"]);
appRunDuration = int.Parse(ConfigurationManager.AppSettings["appRunDuration"]);
rpsFrequency = int.Parse(ConfigurationManager.AppSettings["rpsFrequency"]);
numberOfRowsOfloadLimit = int.Parse(ConfigurationManager.AppSettings["numberOfRowsOfloadLimit"]);
powerBIDesktopPath = ConfigurationManager.AppSettings["powerBIDesktopPath"];

// Initialize Timers
mainTimer.Interval = shockFrequency;
shockTimer.Interval = shockDuration;
rpsTimer.Interval = rpsFrequency;

if (batchSize <= 0) throw new SqlDataGeneratorException("The Batch Size cannot be less or equal to zero.");

if (tasks <= 0) throw new SqlDataGeneratorException("Number Of Tasks cannot be less or equal to zero.");
if (numberOfDataLoadTasks <= 0) throw new SqlDataGeneratorException("Number Of Tasks cannot be less or equal to zero.");

if (delay < 0) throw new SqlDataGeneratorException("Delay cannot be less than zero");
if (dataLoadCommandDelay < 0) throw new SqlDataGeneratorException("Delay cannot be less than zero");

if (meters <= 0) throw new SqlDataGeneratorException("Number Of Meters cannot be less than zero");

if (meters < batchSize * tasks) throw new SqlDataGeneratorException("Number Of Meters cannot be less than (Tasks * BatchSize).");
if (meters < batchSize * numberOfDataLoadTasks) throw new SqlDataGeneratorException("Number Of Meters cannot be less than (Tasks * BatchSize).");

}
catch (Exception exception) { HandleException(exception); }
}
static void mainTimer_Tick(object sender, ElapsedEventArgs e)
{
if (dataGenerator.IsRunning)
{
dataGenerator.Delay = 0;
shockTimer.Start();
}
}
static void rpsTimer_Tick(object sender, ElapsedEventArgs e)
{
try
Expand All @@ -236,11 +230,5 @@ static void rpsTimer_Tick(object sender, ElapsedEventArgs e)
}
catch (Exception exception) { HandleException(exception); }
}
static void shockTimer_Tick(object sender, ElapsedEventArgs e)
{
Random rand = new Random();
dataGenerator.Delay = rand.Next(1500, 3000);
shockTimer.Stop();
}
}
}
Loading

0 comments on commit aaa0465

Please sign in to comment.