We Keep Coding

Event Hub - not getting expected throughput

I have setup an instance of Event Hub with 20 Throughput units and 32 partitions on Standard Tier. As per documentation, every throughput unit equates to 1 MB/second. So ideally I should be getting throughput of 20 MB/second or 1.2 GB/minute. The namespace has only one event hub and I am the only user. The event hub is set up in West US which is the option closest to where requests are sent from.
However, I see that it takes at least 10 minutes to 1.77GB of data. I am using async batch calls and packing each request to the 1 MB limit. I see a vast variance in time taken by SendBatchAsync call - it varies from 0.15 to 25 seconds.
Here is my code :
(Please note : I am constrained to use .Net Framework 4.5)
static EventHubClient eventHubClient;
static Dictionary<int, List<EventData>> events = new Dictionary<int, List<EventData>>();
static Dictionary<int, long> batchSizes = new Dictionary<int, long>();
static long threshold = (long)(1e6 - 1000);
static SemaphoreSlim concurrencySemaphore;
static int maxConcurrency = 1;
static void Main()
{
eventHubClient = EventHubClient.CreateFromConnectionString(connectionString, eventHubName);
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
using (concurrencySemaphore = new SemaphoreSlim(maxConcurrency))
{
foreach (string record in GetRecords())
{
Tuple<int, EventData> currentEventDetails = GetEventData(record);
int partitionId = currentEventDetails.Item1;
EventData currentEvent = currentEventDetails.Item2;
BatchOrSendAsync(partitionId, currentEvent);
}
SendRemainingAsync();
}
stopWatch.Stop();
Console.WriteLine(string.Format("### total time taken = {0}", stopWatch.Elapsed.TotalSeconds.ToString()));
}
static async void BatchOrSendAsync(int partitionId, EventData currentEvent)
{
long batchSize = 0;
batchSizes.TryGetValue(partitionId, out batchSize);
long currentEventSize = currentEvent.SerializedSizeInBytes;
if( batchSize + currentEventSize > threshold)
{
List<EventData> eventsToSend = events[partitionId];
if (eventsToSend == null || eventsToSend.Count == 0)
{
if (currentEventSize > threshold)
throw new Exception("found event with size above threshold");
return;
}
concurrencySemaphore.Wait();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
await eventHubClient.SendBatchAsync(eventsToSend);
stopWatch.Stop();
Console.WriteLine(stopWatch.Elapsed.TotalSeconds.ToString());
concurrencySemaphore.Release();
events[partitionId] = new List<EventData> { currentEvent };
batchSizes[partitionId] = currentEventSize;
}
else
{
if (!events.ContainsKey(partitionId))
{
events[partitionId] = new List<EventData>();
batchSizes[partitionId] = 0;
}
events[partitionId].Add(currentEvent);
batchSizes[partitionId] += currentEventSize;
}
}
static async void SendRemainingAsync()
{
foreach(int partitionId in events.Keys)
{
concurrencySemaphore.Wait();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
await eventHubClient.SendBatchAsync(events[partitionId]);
stopWatch.Stop();
Console.WriteLine(stopWatch.Elapsed.TotalSeconds.ToString());
concurrencySemaphore.Release();
}
}
Note : increasing the maxConcurrency for the semaphore only degrades the overall time taken and the SendBatchAsync call starts erroring out when maxConcurrency is 10
What should I do to improve throughput?

Trying To Pull CPU & Network Usage Information From Performance Monitor

I'm trying to get CPU and network usage information from a process.In my example, I will use the process chromeHere is how im calling the method using an IEnumerable<String>
foreach (string p in GetProcessStatistics(new string[] { "chrome" }))
{
Console.WriteLine(p);
}
And here is the method.
private static IEnumerable<String> GetProcessStatistics(String[] processesTosearch)
{
Process[] processList = Process.GetProcesses();
foreach (string process in processesTosearch)
{
foreach (Process p in processList)
{
if (p.ProcessName == process)
{
StringBuilder sb = new StringBuilder();
PerformanceCounter CPUperformanceCounter = new PerformanceCounter("Process", "% Processor Time", p.ProcessName);
double cpuData = CPUperformanceCounter.NextValue();
PerformanceCounter NETWORKperformanceCounter = new PerformanceCounter("Process", "IO Data Operations/Sec", p.ProcessName);
double networkData = NETWORKperformanceCounter.NextValue();
sb.AppendLine("ID: " + p.Id.ToString());
sb.AppendLine("NAME: " + p.ProcessName);
sb.AppendLine("CPU USAGE: " + cpuData);
sb.AppendLine("RAM USAGE: " + ConvertToReadableSize(p.PrivateMemorySize64));
sb.AppendLine("NETWORK USAGE: " + networkData);
yield return sb.ToString();
}
}
}
}
And here is the output of one of the results
ID: 17624
NAME: chrome
CPU USAGE: 0
RAM USAGE: 23.2MB
NETWORK USAGE: 0
When I look at Performance Monitor, the cpu and network values are not 0, however in the console, they are.I understand from some research that the values will never be perfect, but why are they showing 0 in the console application and not on performance monitor?

You'll need at least two reads for every counter, at least a second apart to get a usable reading.
Rearrange as needed but you would need to do something like this:
private static IEnumerable<String> GetProcessStatistics(String[] processesTosearch)
{
Process[] processList = Process.GetProcesses();
foreach (string process in processesTosearch)
{
foreach (Process p in processList)
{
if (p.ProcessName == process)
{
StringBuilder sb = new StringBuilder();
PerformanceCounter CPUperformanceCounter = new PerformanceCounter("Process", "% Processor Time", p.ProcessName);
PerformanceCounter NETWORKperformanceCounter = new PerformanceCounter("Process", "IO Data Operations/Sec", p.ProcessName);
// set a baseline
CPUperformanceCounter.NextValue();
NETWORKperformanceCounter.NextValue();
Thread.Sleep(1000);
double cpuData = CPUperformanceCounter.NextValue();
double networkData = NETWORKperformanceCounter.NextValue();
sb.AppendLine("ID: " + p.Id.ToString());
sb.AppendLine("NAME: " + p.ProcessName);
sb.AppendLine("CPU USAGE: " + cpuData);
sb.AppendLine("RAM USAGE: " + ConvertToReadableSize(p.PrivateMemorySize64));
sb.AppendLine("NETWORK USAGE: " + networkData);
yield return sb.ToString();
}
}
}
}

I like Jeff's solution, but for me I wanted an average. There were a couple problems getting CPU utilization that seemed like there should be an easy package to solve but I didn't see one.
The first is of course that a value of 0 on the first request is useless. Since you already know that the first response is 0, why doesn't the function just take that into account and return the true .NextValue()?
The second problem is that an instantaneous reading may be wildly inacurrate when trying to make decisions on what resources your app may have available to it since it could be spiking, or between spikes.
My solution was to do a for loop that cycles through and gives you an average for the past few seconds. you can adjust the counter to make it shorter or longer (as long as it is more than 2).
public static float ProcessorUtilization;
public static float GetAverageCPU()
{
PerformanceCounter cpuCounter = new PerformanceCounter("Process", "% Processor Time", Process.GetCurrentProcess().ProcessName);
for (int i = 0; i < 11; ++i)
{
ProcessorUtilization += (cpuCounter.NextValue() / Environment.ProcessorCount);
}
// Remember the first value is 0, so we don't want to average that in.
Console.Writeline(ProcessorUtilization / 10);
return ProcessorUtilization / 10;
}

Speed data processing: ThreadPool or Thread loop with event?

I need to process data in another thread. It can be done in two ways:
Using a thread loop waiting for an event:
AutoResetEvent e = new AutoResetEvent(false)
Thread t = new Thread(delegate
{
while(true)
{
e.WaitOne();
// process data
}
};)
void OnProgramStarted() // one time
{
t.Start();
}
void OnDataReceived()
{
// put data to queue
e.Set();
}
Using the thread pool:
void ProcessData(object state)
{
// process data
}
void OnDataReceived()
{
// put data to queue
ThreadPool.QueueUserWorkItem(ProcessData);
}
What way will be faster?
The real tests give ambiguous results.
My benchmarking code:
using System;
using System.Diagnostics;
using System.Threading;
namespace t_event_tpool
{
class Program
{
const int t = 1000000;
static Stopwatch sw = new Stopwatch();
static int q1, q2;
static AutoResetEvent e1 = new AutoResetEvent(false);
static AutoResetEvent done1 = new AutoResetEvent(false);
static Thread thread = new Thread(ThreadProc);
static void ThreadProc(object state)
{
while(true)
{
e1.WaitOne();
q1++;
done1.Set();
}
}
static AutoResetEvent done2 = new AutoResetEvent(false);
static void PoolProc(object state)
{
q2++;
done2.Set();
}
static void TestA()
{
sw.Restart();
for(int i = 0; i < t; i++)
{
e1.Set();
done1.WaitOne();
}
sw.Stop();
Console.WriteLine("a " + sw.ElapsedMilliseconds + "\t" + q1);
}
static void TestB()
{
sw.Restart();
for(int i = 0; i < t; i++)
{
ThreadPool.QueueUserWorkItem(PoolProc, i);
done2.WaitOne();
}
sw.Stop();
Console.WriteLine("b " + sw.ElapsedMilliseconds + "\t" + q2);
}
static void Main(string[] args)
{
thread.IsBackground = true;
thread.Start();
TestA();
TestB();
TestA();
TestB();
TestA();
TestB();
}
}
}
On low CPU load (without other applications) TestB is 2 times faster than TestA. On high CPU load by other processes TestA faster than TestB.

You are testing the overhead of starting work. This is meaningless for the speed of the actual work. Rather than optimizing overhead, make your work-items more granular so that overhead does not matter. If your work-items run faster than 0.1ms I'd seriously consider making them bigger. In your test, your work-items take just a few instructions.
To answer the question: Your custom thread that is ready to run at the signal of an event has less overhead than posting a work item to the thread-pool. You are doing almost nothing. Hard to improve on that. You'd need to use high-speed queueing patterns like the Disruptor pattern.

How to continuously check for CPU usage in C#?

I mainly followed what was discussed in the second answer to this thread. I want to run a program that will continuously check for CPU usage above 5% for 10 seconds and alert me every time it happens.
How to get the CPU Usage in C#?
And my code is as follows:
static void Main(string[] args)
{
Console.WriteLine("Checking for CPU usage");
int totalhits = 0;
float cpuPercent = getCPUValue();
while (true)
{
if (cpuPercent >= 5)
{
totalhits += 1;
if (totalhits == 10)
{
Console.WriteLine("Alert Usage has exceeded");
Console.WriteLine("Press Enter to continue");
Console.ReadLine();
totalhits = 0;
}
}
else
{
totalhits = 0;
}
}
}
private static float getCPUValue()
{
PerformanceCounter cpuCounter = new PerformanceCounter();
cpuCounter.CategoryName = "Processor";
cpuCounter.CounterName = "% Processor time";
cpuCounter.InstanceName = "_Total";
float firstValue = cpuCounter.NextValue();
System.Threading.Thread.Sleep(50);
float secondValue = cpuCounter.NextValue();
return secondValue;
}
My problem is that it never hits that threshold and if I take out the totalhits = 0; statement inside the innermost if statement then it hits the threshold in less than 5 seconds.
What am I doing wrong?

First of all the
float cpuPercent = getCPUValue();
line should be inside the loop. Otherwise you will read the CPU usage only once. and will iterate over the same value.
You should create Only one PerformanceCounter object, and just call cpuCounter.NextValue() again and again inside the loop.
DON'T create the same CPU PerformanceCounter in every iteration.
PerformanceCounter counter = new PerformanceCounter("Processor", "% Processor Time", "_Total");
while (true)
{
float cpuPercent = counter.nextValue();
if (cpuPercent >= 5)
{
totalhits += 1;
if (totalhits == 10)
{
Console.WriteLine("Alert Usage has exceeded");
Console.WriteLine("Press Enter to continue");
Console.ReadLine();
totalhits = 0;
}
}
else
{
totalhits = 0;
}
}
As stated in MSDN
To obtain performance data for counters that required an initial or previous value for performing the necessary calculation, call the NextValue method twice and use the information returned as your application requires.
So you should call the cpuCounter.NextValue() twice (with about a 1 second delay between the calls) to start getting correct CPU Usage results.
BTW, You should wait about a 1 second between each read operation from the CPU PerformanceCounter (to ensure update).
As shown in this post Retriving Accurate CPU Usate In C#

Use the DispatcherTimer as stated in the below msdn and get the results as you needed.
DispatcherTimer

How to get CPU usage for more than 2 cores?

I try to get in my program CPU usage divided by a cores. Now I use the PerformanceCounter
and changing the InstanceName between 0 and 1 I have the data from 2 cores.
PerformanceCounter pc0 = new PerformanceCounter("Processor", "% Processor Time", "0");
PerformanceCounter pc1 = new PerformanceCounter("Processor", "% Processor Time", "1");
How I can get core usage for 3rd, 4th core etc.?
Does anyone can help me?
Thanks

I suspect that what you are really asking is "How do I count the number of cores?". This code will count the number of cores, and then create performance counters based on that.
int coreCount = 0;
foreach (var item in new System.Management.ManagementObjectSearcher("Select * from Win32_Processor").Get())
{
coreCount += int.Parse(item["NumberOfCores"].ToString());
}
PerformanceCounter[] pc = new PerformanceCounter[coreCount];
for (int i = 0; i < coreCount; i++)
{
pc[i] = new PerformanceCounter("Processor", "% Processor Time", i.ToString());
Console.WriteLine(pc[i].CounterName);
}

I havent used PerformanceCounter before but is there something wrong with doing this?
PerformanceCounter pc0 = new PerformanceCounter("Processor", "% Processor Time", "0");
PerformanceCounter pc1 = new PerformanceCounter("Processor", "% Processor Time", "1");
PerformanceCounter pc2 = new PerformanceCounter("Processor", "% Processor Time", "2");
PerformanceCounter pc3 = new PerformanceCounter("Processor", "% Processor Time", "3");

This might be an old question, but for anyone else looking for a different solution, why don't you use System.Environment?
public static List<System.Diagnostics.PerformanceCounter> GetPerformanceCounters()
{
List<System.Diagnostics.PerformanceCounter> performanceCounters = new List<System.Diagnostics.PerformanceCounter>();
int procCount = System.Environment.ProcessorCount;
for (int i = 0; i < procCount; i++)
{
System.Diagnostics.PerformanceCounter pc = new System.Diagnostics.PerformanceCounter("Processor", "% Processor Time", i.ToString());
performanceCounters.Add(pc);
}
return performanceCounters;
}
EDIT: I noticed this only returns the amount of logical processors, not the actual core count.

Some thing like this should also work for your requirement
public List<string> GetServerStatus()
{
List<string> cpuStatus = new List<string>();
ObjectQuery wmicpus = new WqlObjectQuery("SELECT * FROM Win32_Processor");
ManagementObjectSearcher cpus = new ManagementObjectSearcher(wmicpus);
try
{
int coreCount = 0;
int totusage = 0;
foreach (ManagementObject cpu in cpus.Get())
{
//cpuStatus.Add(cpu["DeviceId"] + " = " + cpu["LoadPercentage"]);
coreCount += 1;
totusage += Convert.ToInt32(cpu["LoadPercentage"]);
}
if (coreCount > 1)
{
double ActUtiFloat = totusage / coreCount;
int ActUti = Convert.ToInt32(Math.Round(ActUtiFloat));
//Utilisation = ActUti + "%";
cpuStatus.Add("CPU = " + ActUti);
}
else
{
cpuStatus.Add("CPU = " + totusage);
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
cpus.Dispose();
}
return cpuStatus;
}

foreach (var item in new System.Management.ManagementObjectSearcher("Select NumberOfLogicalProcessors from Win32_Processor").Get())
coreCount += int.Parse(item["NumberOfLogicalProcessors"].ToString());
PerformanceCounter[] pc = new PerformanceCounter[coreCount];
for (int i = 0; i < coreCount; i++)
pc[i] = new PerformanceCounter("Processor", "% Processor Time", i.ToString());