Related
Context
I am importing IMDB database files into an SQLite database with the help of EntityFrameworkCore. In fact, two files, the titles.basics and the titles.akas (which is linked to basics via its movie ID).
At first, I had a single thread reading lines from basics and loop through akas until it changes of ID. Though, there was an issue there and most of all, it was too slow. So, I decided to create a multithread code that would read both files at the same time and another combining akas with the appropriate movie.
I am currently importing so I still do not know if my issue is fixed (probably it is). Though, it is still too much slow for me.
Issue
The combining part is still very slow, but more importantly, I can see my process is only using around 12% of CPU which corresponds to only 1/8 of total usage and I have 8 physical cores. So, it really seems the process is only using 1 core.
I am not giving any code here, as having a minimal testable code wouldn't mean anything. Though, you can see both versions here:
https://cints.net/public/Imdb-MultiThread.cs.txt
using com.cyberinternauts.all.MediaRecognizer.Database;
using com.cyberinternauts.all.MediaRecognizer.Models.Metas;
using Microsoft.EntityFrameworkCore;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
namespace com.cyberinternauts.all.MediaRecognizer.MetaSources
{
class Imdb : MediaSource
{
private const string TITLES_FILE = "title.basics.tsv.gz";
private const string AKAS_FILE = "title.akas.tsv.gz";
private readonly string temporaryFolder = #"c:\temp\";
private readonly string baseUrl = "https://datasets.imdbws.com/";
private readonly WebClient webClient = new();
MediaRecognizerContext db = new();
private IQueryable<MetaMovie> imdbMovies = null;
private async Task<bool> GatherFilesAsync()
{
var totalFilesGathered = 0;
var filesToDownload = new string[] { AKAS_FILE, TITLES_FILE };
foreach(var fileToDownload in filesToDownload)
{
var compressedFile = temporaryFolder + fileToDownload;
if (!File.Exists(compressedFile) || !File.GetLastWriteTime(compressedFile).Date.Equals(DateTime.Today))
{
await GatherFileAsync(fileToDownload);
totalFilesGathered++;
}
}
return totalFilesGathered != 0;
}
private async Task GatherFileAsync(string fileName)
{
var compressedFile = temporaryFolder + fileName;
var uncompressedFile = temporaryFolder + Path.GetFileNameWithoutExtension(compressedFile);
await webClient.DownloadFileTaskAsync(baseUrl + fileName, compressedFile);
using Stream fd = File.Create(uncompressedFile);
using Stream fs = File.OpenRead(compressedFile);
using Stream csStream = new GZipStream(fs, CompressionMode.Decompress);
var buffer = new byte[1024];
int nRead;
while ((nRead = await csStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
await fd.WriteAsync(buffer, 0, nRead);
}
}
private async Task LoadMetaDataAsync()
{
//return; //TODO: Remove this line
//TODO: Reactivate this line
//if (!await GatherFilesAsync()) return;
var titlesFile = temporaryFolder + Path.GetFileNameWithoutExtension(TITLES_FILE);
var akasFile = temporaryFolder + Path.GetFileNameWithoutExtension(AKAS_FILE);
var dbLock = new SemaphoreSlim(1);
var akasLock = new SemaphoreSlim(1);
var currentTitlesAkasLock = new SemaphoreSlim(1);
var associateLock = new SemaphoreSlim(1);
using (var db = new MediaRecognizerContext())
{
db.ChangeTracker.AutoDetectChangesEnabled = false;
var titles = new ConcurrentDictionary<string, MetaMovie>();
var readTitles = Task.Factory.StartNew(() =>
{
Parallel.ForEach(File.ReadLines(titlesFile), (titleLine, _, readingIndex) =>
{
if (readingIndex == 0) return; // Skipping columns titles line
var movieInfos = titleLine.Split("\t", StringSplitOptions.None);
dbLock.Wait();
MetaMovie metaMovie = db.MetaMovies.Where(m => m.ExternalId == movieInfos[0]).Include(m => m.Titles).FirstOrDefault();
dbLock.Release();
if (metaMovie == null)
{
int totalMinutes = -1;
if (!int.TryParse(movieInfos[7], out totalMinutes))
{
totalMinutes = -1;
}
metaMovie = new MetaMovie
{
ExternalId = movieInfos[0],
MetaSource = nameof(Imdb),
MovieType = movieInfos[1],
Title = movieInfos[3],
TotalMinutes = totalMinutes,
Genres = movieInfos[8]
};
metaMovie.Titles = new List<MetaTitle>();
if (int.TryParse(movieInfos[5], out int startYear))
{
metaMovie.StartYear = new DateTime(startYear, 1, 1);
}
else
{
metaMovie.StartYear = new DateTime(9999, 1, 1);
}
if (int.TryParse(movieInfos[6], out int endYear))
{
metaMovie.EndYear = new DateTime(endYear, 1, 1);
}
else
{
metaMovie.EndYear = metaMovie.StartYear;
}
}
titles.TryAdd(metaMovie.ExternalId, metaMovie);
});
});
var akas = new Dictionary<string, List<MetaTitle>>();
var currentTitlesAkas = new ConcurrentDictionary<string, int>();
var readAkas = Task.Factory.StartNew(() =>
{
Parallel.ForEach(File.ReadLines(akasFile), (akaLine, _, readingIndex) =>
{
if (readingIndex == 0) return; // Skipping columns titles line
currentTitlesAkasLock.Wait();
var titleInfos = akaLine.Split("\t", StringSplitOptions.None);
var externalId = titleInfos[0];
if (!currentTitlesAkas.ContainsKey(externalId))
{
currentTitlesAkas.TryAdd(externalId, 1);
}
else
{
currentTitlesAkas[externalId]++;
}
currentTitlesAkasLock.Release();
var metaTitle = new MetaTitle
{
MetaMovie = null,
Text = titleInfos[2],
Region = titleInfos[3],
Language = titleInfos[4]
};
akasLock.Wait();
List<MetaTitle> titleAkas;
if (!akas.ContainsKey(externalId))
{
titleAkas = new List<MetaTitle>();
akas.Add(externalId, titleAkas);
}
else
{
titleAkas = akas[externalId];
}
titleAkas.Add(metaTitle);
akasLock.Release();
currentTitlesAkasLock.Wait();
currentTitlesAkas[externalId]--;
currentTitlesAkasLock.Release();
});
});
var savingCounter = 0;
var associate = Task.Factory.StartNew(() =>
{
Parallel.For(1, Environment.ProcessorCount * 10, async (_) =>
{
var isAssociating = true;
do
{
var externalId = string.Empty;
var currentTitleAkaRemoved = false;
currentTitlesAkasLock.Wait();
foreach (var curExternalId in currentTitlesAkas.Keys.OrderBy(t => t))
{
if (currentTitlesAkas[curExternalId] == 0)
{
externalId = curExternalId;
break;
}
}
if (externalId != String.Empty)
{
currentTitleAkaRemoved = currentTitlesAkas.TryRemove(externalId, out int useless0); // Removing so other threads won't take it
}
isAssociating = !readAkas.IsCompleted || !readTitles.IsCompleted || !currentTitlesAkas.IsEmpty;
currentTitlesAkasLock.Release();
if (String.IsNullOrEmpty(externalId) || !currentTitleAkaRemoved) continue;
if (titles.TryGetValue(externalId, out MetaMovie metaMovie))
{
akasLock.Wait();
var titleAkas = akas[externalId];
akas.Remove(externalId);
akasLock.Release();
var changedMovie = false;
var movieAkas = metaMovie.Titles.Select(t => t).ToList(); // Clone list
foreach (var metaTitle in titleAkas)
{
var existingTitle = movieAkas.Where(t => t.Text == metaTitle.Text && t.Region == metaTitle.Region && t.Language == metaTitle.Language).FirstOrDefault();
if (existingTitle == null)
{
changedMovie = true;
metaMovie.Titles.Add(metaTitle);
}
else
{
movieAkas.Remove(existingTitle);
}
}
foreach (var movieTitle in movieAkas)
{
changedMovie = true;
metaMovie.Titles.Remove(movieTitle);
}
dbLock.Wait();
if (metaMovie.Id == 0)
{
db.Add(metaMovie);
}
else if (changedMovie)
{
db.Update(metaMovie);
}
dbLock.Release();
currentTitlesAkasLock.Wait();
currentTitlesAkas.TryRemove(externalId, out int uselessOut); // Free memory
isAssociating = !readAkas.IsCompleted || !readTitles.IsCompleted || !currentTitlesAkas.IsEmpty;
currentTitlesAkasLock.Release();
titles.TryRemove(externalId, out MetaMovie uselessOut2); // Free memory
associateLock.Wait();
savingCounter++;
var localSavingCounter = savingCounter;
associateLock.Release();
if (localSavingCounter != 0 && localSavingCounter % 1000 == 0)
{
var ttt = currentTitlesAkas.Where(t => t.Value > 0);
dbLock.Wait();
await db.SaveChangesAsync();
dbLock.Release();
Console.WriteLine("Saved " + localSavingCounter);
}
}
else if (!readTitles.IsCompleted) // If reading titles is not ended, then maybe it was not read yet... otherwise, it doesn't exist
{
currentTitlesAkasLock.Wait();
currentTitlesAkas.TryAdd(externalId, 0); // Readd because still no movie associated
currentTitlesAkasLock.Release();
}
} while (isAssociating);
});
});
Task.WaitAll(readTitles, readAkas, associate);
await db.SaveChangesAsync();
}
}
public async override Task<IEnumerable<MetaMovie>> FindMediasAsync(DirectoryInfo directory)
{
await LoadMetaDataAsync();
var movie = await ExtractInfosAsync(directory);
if (movie == null) return null;
if (imdbMovies == null)
{
imdbMovies = db.MetaMovies.Where(m => m.MetaSource == nameof(Imdb) && m.MovieType == "movie");
}
return FindCorrespondances(imdbMovies, movie);
}
}
}
https://cints.net/public/Imdb-SingleThread.cs.txt
using com.cyberinternauts.all.MediaRecognizer.Database;
using com.cyberinternauts.all.MediaRecognizer.Models.Metas;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
namespace com.cyberinternauts.all.MediaRecognizer.MetaSources
{
class Imdb : MediaSource
{
private const string TITLES_FILE = "title.basics.tsv.gz";
private const string AKAS_FILE = "title.akas.tsv.gz";
private readonly string temporaryFolder = #"c:\temp\";
private readonly string baseUrl = "https://datasets.imdbws.com/";
private readonly WebClient webClient = new();
MediaRecognizerContext db = new();
private IQueryable<MetaMovie> imdbMovies = null;
private async Task<bool> GatherFilesAsync()
{
var totalFilesGathered = 0;
var filesToDownload = new string[] { AKAS_FILE, TITLES_FILE };
foreach(var fileToDownload in filesToDownload)
{
var compressedFile = temporaryFolder + fileToDownload;
if (!File.Exists(compressedFile) || !File.GetLastWriteTime(compressedFile).Date.Equals(DateTime.Today))
{
await GatherFileAsync(fileToDownload);
totalFilesGathered++;
}
}
return totalFilesGathered != 0;
}
private async Task GatherFileAsync(string fileName)
{
var compressedFile = temporaryFolder + fileName;
var uncompressedFile = temporaryFolder + Path.GetFileNameWithoutExtension(compressedFile);
await webClient.DownloadFileTaskAsync(baseUrl + fileName, compressedFile);
using Stream fd = File.Create(uncompressedFile);
using Stream fs = File.OpenRead(compressedFile);
using Stream csStream = new GZipStream(fs, CompressionMode.Decompress);
var buffer = new byte[1024];
int nRead;
while ((nRead = await csStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
await fd.WriteAsync(buffer, 0, nRead);
}
}
private async Task LoadMetaDataAsync()
{
//return; //TODO: Remove this line
//TODO: Reactivate this line
//if (!await GatherFilesAsync()) return;
var titlesFile = temporaryFolder + Path.GetFileNameWithoutExtension(TITLES_FILE);
var akasFile = temporaryFolder + Path.GetFileNameWithoutExtension(AKAS_FILE);
var titlesLines = File.ReadLines(titlesFile);
var akasLines = File.ReadLines(akasFile);
var titlesIterator = titlesLines.GetEnumerator();
titlesIterator.MoveNext(); // Skip columns headers
var akasIterator = akasLines.GetEnumerator();
akasIterator.MoveNext();
akasIterator.MoveNext(); // Done twice to skip columns headers
var currentAka = akasIterator.Current;
var savingCounter = 0;
using (var db = new MediaRecognizerContext())
{
db.ChangeTracker.AutoDetectChangesEnabled = false;
while (titlesIterator.MoveNext())
{
var titleLine = titlesIterator.Current;
var movieInfos = titleLine.Split("\t", StringSplitOptions.None);
MetaMovie metaMovie = db.MetaMovies.Where(m => m.ExternalId == movieInfos[0]).FirstOrDefault();
var isNewMovie = false;
if (metaMovie == null)
{
int totalMinutes = -1;
if (!int.TryParse(movieInfos[7], out totalMinutes))
{
totalMinutes = -1;
}
isNewMovie = true;
metaMovie = new MetaMovie
{
ExternalId = movieInfos[0],
MetaSource = nameof(Imdb),
MovieType = movieInfos[1],
Title = movieInfos[3],
TotalMinutes = totalMinutes,
Genres = movieInfos[8]
};
metaMovie.Titles = new List<MetaTitle>();
if (int.TryParse(movieInfos[5], out int startYear))
{
metaMovie.StartYear = new DateTime(startYear, 1, 1);
}
else
{
metaMovie.StartYear = new DateTime(9999, 1, 1);
}
if (int.TryParse(movieInfos[6], out int endYear))
{
metaMovie.EndYear = new DateTime(endYear, 1, 1);
}
else
{
metaMovie.EndYear = metaMovie.StartYear;
}
}
var movieAkasIds = metaMovie.Titles.Select(t => t.Id).ToList();
var titleInfos = currentAka?.Split("\t", StringSplitOptions.None);
while (currentAka != null && int.Parse(titleInfos[0][2..]) <= int.Parse(metaMovie.ExternalId[2..]))
{
if (titleInfos[0] == metaMovie.ExternalId)
{
var metaTitle = new MetaTitle
{
MetaMovie = metaMovie,
Text = titleInfos[2],
Region = titleInfos[3],
Language = titleInfos[4]
};
var existingTitle = metaMovie.Titles.Where(t => t.Text == metaTitle.Text && t.Region == metaTitle.Region && t.Language == metaTitle.Language).FirstOrDefault();
if (existingTitle == null)
{
metaMovie.Titles.Add(metaTitle);
}
else
{
movieAkasIds.Remove(existingTitle.Id);
}
}
else
{
var a = 1;
}
akasIterator.MoveNext();
currentAka = akasIterator.Current;
titleInfos = currentAka.Split("\t", StringSplitOptions.None);
}
foreach(var movieTitleId in movieAkasIds)
{
metaMovie.Titles.Remove(metaMovie.Titles.Where(t => t.Id == movieTitleId).FirstOrDefault());
}
if (isNewMovie)
{
db.Add(metaMovie);
}
else
{
db.Update(metaMovie);
}
savingCounter++;
if (savingCounter % 10000 == 0)
{
await db.SaveChangesAsync();
Console.WriteLine("Saved " + savingCounter);
}
}
await db.SaveChangesAsync();
}
}
public async override Task<IEnumerable<MetaMovie>> FindMediasAsync(DirectoryInfo directory)
{
await LoadMetaDataAsync();
var movie = await ExtractInfosAsync(directory);
if (movie == null) return null;
if (imdbMovies == null)
{
imdbMovies = db.MetaMovies.Where(m => m.MetaSource == nameof(Imdb) && m.MovieType == "movie");
}
return FindCorrespondances(imdbMovies, movie);
}
}
}
In the multithread version, the slow part is in the method LoadMetaDataAsync and more precisely in var associate = Task.Factory.StartNew(() => code part.
This is in development and cleaning, splitting will done after I have the appropriate result/speed.
Case closed. I returned to the single thread version and I found my initial issue (my code was supposing the files were in order, which they were partially).
Thank you for all people that participated.
I have a program that needs to scan for other devices running my program on the network. The solution I came up with was to call each ipAddress to see if my program is running.
The code below is completely blocking the cpu:-
using Newtonsoft.Json;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace FileWire
{
class SearchNearby
{
private bool pc_search_cancelled = false;
private SynchronizedCollection<Thread> PCSearchThreadList;
private ConcurrentDictionary<String, String> NearbyPCList;
public void NewPcFound(string s, string s1)
{
Console.WriteLine(string.Format("PC Found at: {0} PC Name: {1}", s, s1));
}
public SearchNearby()
{
startPCScan();
while (true)
{
bool isAnyAlive = false;
foreach(Thread t in PCSearchThreadList)
{
isAnyAlive |= t.IsAlive;
}
if (!isAnyAlive)
{
Console.WriteLine("Search Complete");
foreach (var a in NearbyPCList)
{
Console.WriteLine(a.Key + " ;; " + a.Value);
}
startPCScan();
}
Thread.Sleep(100);
}
}
private void startPCScan()
{
PCSearchThreadList = new SynchronizedCollection<Thread>();
NearbyPCList = new ConcurrentDictionary<String, String>();
pc_search_cancelled = false;
String add = "";
System.Net.IPAddress[] ad = System.Net.Dns.GetHostByName(System.Net.Dns.GetHostName()).AddressList;
foreach (System.Net.IPAddress ip in ad)
{
add += ip.ToString() + "\n";
}
bool connected;
if (add.Trim(' ').Length == 0)
{
connected = false;
}
else
{
connected = true;
}
if (connected)
{
try
{
String[] addresses = add.Split('\n');
foreach (String address in addresses)
{
int myIP = int.Parse(address.Substring(address.LastIndexOf(".") + 1));
for (int def = 0; def <= 10; def++)
{
int finalDef = def;
for (int j = 0; j < 10; j++)
{
string finalJ = j.ToString();
Thread thread = new Thread(new ThreadStart(() =>
{
if (!pc_search_cancelled)
{
for (int i = (finalDef * 25); i < (finalDef * 25) + 25 && i <= 255; i++)
{
if (!pc_search_cancelled)
{
if (i != myIP)
{
String callToAddress = "http://" + address.Substring(0, address.LastIndexOf(".")) + "." + i + ":" + (1234 + int.Parse(finalJ)).ToString();
String name = canGetNameAndAvatar(callToAddress);
if (name != null)
{
NearbyPCList[callToAddress] = name;
NewPcFound(callToAddress, name);
}
}
}
}
}
}));
PCSearchThreadList.Add(thread);
thread.Start();
}
}
}
} catch (Exception e) {
}
}
}
private String canGetNameAndAvatar(String connection)
{
String link = connection + "/getAvatarAndName";
link = link.Replace(" ", "%20");
try
{
var client = new HttpClient();
client.Timeout = TimeSpan.FromMilliseconds(500);
var a = new Task<HttpResponseMessage>[1];
a[0] = client.GetAsync(link);
Task.WaitAll(a);
var b = a[0].Result.Content.ReadAsStringAsync();
Task.WaitAll(b);
Console.WriteLine(b.Result);
string result = b.Result;
result = result.Substring(result.IndexOf("<body>") + 6, result.IndexOf("</body>") - (result.IndexOf("<body>") + 6));
AvtarAndName json = JsonConvert.DeserializeObject<AvtarAndName>(result);
if (json != null)
{
return json.name;
}
}
catch
{
return null;
}
return null;
}
}
}
This is the exact C# version of the java code I was using in Java:-
import com.sun.istack.internal.Nullable;
import org.apache.http.*;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.HttpParams;
import org.json.JSONObject;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.URI;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
public class PCScan {
private static boolean pc_search_cancelled = false;
private static List<Thread> PCSearchThreadList;
private static HashMapWithListener<String, String> NearbyPCList;
public static void main(String[] args) {
start();
while (true) {
int numCompleted = 0;
for (Thread t : PCSearchThreadList) {
if (!t.isAlive()) {
numCompleted++;
}
}
if (numCompleted == PCSearchThreadList.size()) {
start();
}
}
}
private static void start() {
try {
startPCScan();
} catch (SocketException e) {
e.printStackTrace();
}
NearbyPCList.setPutListener(new HashMapWithListener.putListener() {
#Override
public void onPut(Object key, Object value) {
System.out.println(key.toString() + ";;" + value.toString());
}
});
}
private static void startPCScan() throws SocketException {
pc_search_cancelled = false;
PCSearchThreadList = new CopyOnWriteArrayList<>();
NearbyPCList = new HashMapWithListener<>();
Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();
boolean connected;
String add = "";
while (enumeration.hasMoreElements()) {
NetworkInterface interfacea = enumeration.nextElement();
if (!interfacea.isLoopback()) {
Enumeration<InetAddress> enumeration1 = interfacea.getInetAddresses();
while (enumeration1.hasMoreElements()) {
String address = enumeration1.nextElement().getHostAddress();
if (address.split("\\.").length == 4) {
add += address + "\n";
}
}
}
}
System.out.println(add);
connected = true;
if (connected) {
try {
String[] addresses = add.split("\n");
addresses = new HashSet<String>(Arrays.asList(addresses)).toArray(new String[0]);
for (String address : addresses) {
int myIP = Integer.parseInt(address.substring(address.lastIndexOf(".") + 1));
for (int def = 0; def <= 10; def++) {
int finalDef = def;
for (int j = 0; j < 10; j++) {
int finalJ = j;
Thread thread = new Thread(new Runnable() {
#Override
public void run() {
if (!pc_search_cancelled) {
for (int i = (finalDef * 25); i < (finalDef * 25) + 25 && i <= 255; i++) {
if (!pc_search_cancelled) {
if (i != myIP) {
String callToAddress = "http://" + address.substring(0, address.lastIndexOf(".")) + "." + i + ":" + String.valueOf(Integer.parseInt("1234") + finalJ);
String name = canGetNameAndAvatar(callToAddress);
if (name != null) {
NearbyPCList.put(callToAddress, name);
}
}
}
}
}
}
});
PCSearchThreadList.add(thread);
thread.start();
}
}
// }
// }).start();
}
} catch (Exception e) {
}
}
}
private static String canGetNameAndAvatar(String connection) {
String link = connection + "/getAvatarAndName";
link = link.replaceAll(" ", "%20");
try {
HttpClient client = new DefaultHttpClient();
HttpParams httpParams = client.getParams();
httpParams.setParameter(
CoreConnectionPNames.CONNECTION_TIMEOUT, 500);
HttpGet request = new HttpGet();
request.setURI(new URI(link));
HttpResponse response = client.execute(request);
BufferedReader in = new BufferedReader(new
InputStreamReader(response.getEntity().getContent()));
StringBuffer sb = new StringBuffer("");
String line="";
while ((line = in.readLine()) != null) {
sb.append(line);
break;
}
in.close();
String result = sb.toString();
result = result.substring(result.indexOf("<body>") + 6, result.indexOf("</body>"));
JSONObject json = new JSONObject(result);
if (json != null) {
return json.getString("name");
}
}
catch (Exception ignored){
return null;
}
return null;
}
static class HashMapWithListener<K, V> extends HashMap<K, V> {
private putListener PutListener;
public void setPutListener(putListener PutListener) {
this.PutListener = PutListener;
}
#Nullable
#Override
public V put(K key, V value) {
PutListener.onPut(key, value);
return super.put(key, value);
}
interface putListener {
public void onPut(Object key, Object value);
}
}
}
The java code runs absolutely fine and only uses about 20 percent cpu while c# code absolutely locks the PC. I tried Webclient, webrequest, httpClient. All have literally the same performance.
I need the code to be in c# as I can't include whole JRE in my program since it is too large. The rest of my program and GUI is in WPF format.
Also, I need the code to take a maximum of 50seconds while scanning ports 1234-1243. This code also works absolutely fine even on a midrange android phone. So, I don't know what the problem is.
I would suggest something like this (I've simplified it for the sake of an example):
private static HttpClient _client = new HttpClient() { Timeout = TimeSpan.FromMilliseconds(500) };
private async Task<Something> GetSomething(string url)
{
using (HttpResponseMessage response = await _client.GetAsync(url))
{
string json = await response.ReadAsStringAsync();
return JsonConvert.DeserializeObject<Something>(json);
}
}
private async Task<Something[]> GetSomethings(string[] urls)
{
IEnumerable<Task<Something>> requestTasks = urls.Select(u => GetSomething(u));
Something[] results = await Task.WhenAll<Something>(requestTasks);
return results;
}
You should also make the method calling GetSomethings async, and await it, and do the same all the way up the call chain.
async/await uses a thread pool to execute, and the thread is actually suspended while the IO part of the request occurs, meaning that no CPU time is used during this period. When then IO part is done, it resumes the code at the await.
Related information:
Asynchronous programming documentation
How and when to use 'async' and 'await'
You are using threads and multithreading completely wrong.
Because I cannot really understand what you are trying to do because everything is cramped into one function, I cannot provide you with a more detailed solution.
But let me suggest the following: I understand, you want to execute some operation in the background connecting to some other computer, try something like this
var taskList = new List<Task>();
foreach (var pc in computers)
{
var currentPcTask = Task.Run(() => DoYourWorkForSomePcHere(pc));
taskList.Add(currentPcTask);
}
Task.WaitAll(taskList.ToArray());
This will be very CPU efficient.
I am developing a UWP VPN Plugin. In later stage it should handle OpenVPN. In the first stage I am trying to understand the VpnPlugin to get it work in the simplest possible way. For testing I am using Android's ToyVpn test Server on a Debian VM (https://android.googlesource.com/platform/development/+/master/samples/ToyVpn/server/linux). Unfortunately the VpnPlugin is poor and lousy documented, no Guideline - no Nothing. The Github examples are useless and not working either, even https://github.com/ysc3839/UWPToyVpn gives only Rough orientation. I was able to successfully do the Handshake with the Server who responds with a Parameter chain. When it Comes to start the Connection, an exception is thrown that the device is not connected. I am running out of Options and any help would greatly appreciated.
public sealed class ToyVpnPlugin : IVpnPlugIn
{
DatagramSocket _datagramSocket;
public async void Connect(VpnChannel channel)
{
//string parameters = default;
string serverPort = "8000";
string secret = "test";
_datagramSocket = new DatagramSocket();
_datagramSocket.MessageReceived += (s, e) =>
{
DataReader dataReader = e.GetDataReader();
if (dataReader.UnconsumedBufferLength > 0 && dataReader.ReadByte() == 0)
{
var parameters = dataReader.ReadString(dataReader.UnconsumedBufferLength);
ConfigureAndConnect(channel, parameters);
}
};
var serverHostName = channel.Configuration.ServerHostNameList[0];
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.LoadXml(channel.Configuration.CustomField);
var firstChild = xmlDocument.FirstChild;
if (firstChild.Name.Equals("ToyVpnConfig"))
{
foreach (XmlNode childNode in firstChild.ChildNodes)
{
if (childNode.Name.Equals("ServerPort")) serverPort = childNode.InnerText;
else if (childNode.Name.Equals("Secret")) secret = childNode.InnerText;
}
}
await _datagramSocket.ConnectAsync(serverHostName, serverPort);
await HandShake(_datagramSocket, secret);
}
public void Disconnect(VpnChannel channel)
{
channel.Stop();
}
public void GetKeepAlivePayload(VpnChannel channel, out VpnPacketBuffer keepAlivePacket)
{
keepAlivePacket = null;
}
public void Encapsulate(VpnChannel channel, VpnPacketBufferList packets, VpnPacketBufferList encapulatedPackets)
{
while (packets.Size > 0)
{
VpnPacketBuffer vpnPacketBuffer = packets.RemoveAtBegin();
var buffer = vpnPacketBuffer.Buffer;
VpnPacketBufferStatus vpnPacketBufferStatus = vpnPacketBuffer.Status;
encapulatedPackets.Append(vpnPacketBuffer);
}
}
public void Decapsulate(VpnChannel channel, VpnPacketBuffer encapBuffer, VpnPacketBufferList decapsulatedPackets, VpnPacketBufferList controlPacketsToSend)
{
while (encapBuffer != null)
{
decapsulatedPackets.Append(encapBuffer);
}
}
async Task HandShake(DatagramSocket datagramSocket, string secret)
{
for (int i = 0; i < 3; i++)
{
var dataWriter = new DataWriter(datagramSocket.OutputStream)
{
UnicodeEncoding = Windows.Storage.Streams.UnicodeEncoding.Utf8
};
dataWriter.WriteByte(0);
dataWriter.WriteString(secret);
await dataWriter.StoreAsync();
dataWriter.DetachStream();
}
}
void ConfigureAndConnect(VpnChannel vpnChannel, string parameters)
{
parameters = parameters.TrimEnd();
uint mtu = 1500;
List<HostName> ipv4InclusionHostNames = new List<HostName>();
List<HostName> dnsServerHostNames = new List<HostName>();
VpnRouteAssignment vpnRouteAssignment = new VpnRouteAssignment();
var ipv4InclusionRoutes = vpnRouteAssignment.Ipv4InclusionRoutes;
foreach (var parameter in parameters.Split(null))
{
var fields = parameter.Split(",");
try
{
switch (fields[0])
{
case "m":
mtu = uint.Parse(fields[1]);
break;
case "a":
ipv4InclusionHostNames.Add(new HostName(fields[1]));
break;
case "r":
ipv4InclusionRoutes.Add(new VpnRoute(new HostName(fields[1]), (byte)uint.Parse(fields[2])));
break;
case "d":
dnsServerHostNames.Add(new HostName(fields[1]));
break;
case "s":
//TODO "SearchDomain"
break;
default:
break;
}
}
catch (Exception)
{
throw;
}
}
VpnDomainNameAssignment vpnDomainNameAssignment = new VpnDomainNameAssignment();
vpnDomainNameAssignment.DomainNameList.Add(new VpnDomainNameInfo(".", VpnDomainNameType.Suffix, dnsServerHostNames, null));
try
{
vpnChannel.AssociateTransport(_datagramSocket, null);
vpnChannel.StartExistingTransports(ipv4InclusionHostNames, null, null, vpnRouteAssignment, vpnDomainNameAssignment, mtu, 65535, false);
}
catch (Exception e)
{
vpnChannel.TerminateConnection(e.Message);
}
}
}
This appears to be unconfirmed working, any better idea or suggestions are greatly appreciated as I figured this out by more or less "fishing"...:
namespace Background
{
public enum HandshakeState { Waiting, Received, Canceled };
public sealed class SecurepointVpnPlugin : IVpnPlugIn
{
DatagramSocket _datagramSocket;
HandshakeState _handshakeState;
public void Connect(VpnChannel channel)
{
string serverPort = "8000";
string secret = "test";
string parameters = null;
_datagramSocket = new DatagramSocket();
channel.AssociateTransport(_datagramSocket, null);
_datagramSocket.MessageReceived += (s, e) =>
{
DataReader dataReader = e.GetDataReader();
if (dataReader.UnconsumedBufferLength > 0 && dataReader.ReadByte() == 0)
{
parameters = dataReader.ReadString(dataReader.UnconsumedBufferLength);
_handshakeState = HandshakeState.Received;
}
};
var serverHostName = channel.Configuration.ServerHostNameList[0];
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.LoadXml(channel.Configuration.CustomField);
var firstChild = xmlDocument.FirstChild;
if (firstChild.Name.Equals("ToyVpnConfig"))
{
foreach (XmlNode childNode in firstChild.ChildNodes)
{
if (childNode.Name.Equals("ServerPort")) serverPort = childNode.InnerText;
else if (childNode.Name.Equals("Secret")) secret = childNode.InnerText;
}
}
_datagramSocket.ConnectAsync(serverHostName, serverPort).AsTask().GetAwaiter().GetResult();
_handshakeState = HandshakeState.Waiting;
HandShake(_datagramSocket, secret).AsTask().GetAwaiter().GetResult();
if (_handshakeState == HandshakeState.Received) ConfigureAndConnect(channel, parameters);
else channel.Stop();
}
public void Disconnect(VpnChannel channel)
{
channel.Stop();
}
public void GetKeepAlivePayload(VpnChannel channel, out VpnPacketBuffer keepAlivePacket)
{
keepAlivePacket = null;
}
public void Encapsulate(VpnChannel channel, VpnPacketBufferList packets, VpnPacketBufferList encapulatedPackets)
{
while (packets.Size > 0) encapulatedPackets.Append(packets.RemoveAtBegin());
}
public void Decapsulate(VpnChannel channel, VpnPacketBuffer encapBuffer, VpnPacketBufferList decapsulatedPackets, VpnPacketBufferList controlPacketsToSend)
{
decapsulatedPackets.Append(encapBuffer);
}
IAsyncAction HandShake(DatagramSocket datagramSocket, string secret)
{
return Task.Run(async () =>
{
for (int i = 0; i < 3; i++)
{
var dataWriter = new DataWriter(datagramSocket.OutputStream)
{
UnicodeEncoding = Windows.Storage.Streams.UnicodeEncoding.Utf8
};
dataWriter.WriteByte(0);
dataWriter.WriteString(secret);
await dataWriter.StoreAsync();
dataWriter.DetachStream();
}
for (int i = 0; i < 50; i++)
{
await Task.Delay(100);
switch (_handshakeState)
{
case HandshakeState.Waiting:
break;
case HandshakeState.Received:
return;
case HandshakeState.Canceled:
throw new OperationCanceledException();
default:
break;
}
}
}).AsAsyncAction();
}
void ConfigureAndConnect(VpnChannel vpnChannel, string parameters)
{
parameters = parameters.TrimEnd();
uint mtuSize = 68;
var assignedClientIPv4list = new List<HostName>();
var dnsServerList = new List<HostName>();
VpnRouteAssignment assignedRoutes = new VpnRouteAssignment();
VpnDomainNameAssignment assignedDomainName = new VpnDomainNameAssignment();
var ipv4InclusionRoutes = assignedRoutes.Ipv4InclusionRoutes;
foreach (var parameter in parameters.Split(null))
{
var fields = parameter.Split(",");
switch (fields[0])
{
case "m":
mtuSize = uint.Parse(fields[1]);
break;
case "a":
assignedClientIPv4list.Add(new HostName(fields[1]));
break;
case "r":
ipv4InclusionRoutes.Add(new VpnRoute(new HostName(fields[1]), (byte)(int.Parse(fields[2]))));
break;
case "d":
dnsServerList.Add(new HostName(fields[1]));
break;
default:
break;
}
}
assignedRoutes.Ipv4InclusionRoutes = ipv4InclusionRoutes;
assignedDomainName.DomainNameList.Add(new VpnDomainNameInfo(".", VpnDomainNameType.Suffix, dnsServerList, null));
try
{
vpnChannel.StartExistingTransports(assignedClientIPv4list, null, null, assignedRoutes, assignedDomainName, mtuSize, mtuSize + 18, false);
}
catch (Exception e)
{
vpnChannel.TerminateConnection(e.Message);
}
}
}
}
I read this article , I want to get sent bytes and receive bytes for all process, but sent bytes and receive bytes of all process always return 0;
Here is my code:
namespace Network
{
public class NetworkTraffic
{
private List<PerformanceCounter> listBytesSentPerformanceCounter;
private List<PerformanceCounter> listBytesReceivedPerformanceCounter;
//private PerformanceCounter bytesSentPerformanceCounter;
//private PerformanceCounter bytesReceivedPerformanceCounter;
public NetworkTraffic()
{
listBytesSentPerformanceCounter = new List<PerformanceCounter>();
listBytesReceivedPerformanceCounter = new List<PerformanceCounter>();
List<string> listInstanceName = GetInstanceName();
foreach (string str in listInstanceName)
{
PerformanceCounter bytesSentPerformanceCounter = new PerformanceCounter();
PerformanceCounter bytesReceivedPerformanceCounter = new PerformanceCounter();
bytesSentPerformanceCounter.CategoryName = ".NET CLR Networking 4.0.0.0";
bytesSentPerformanceCounter.CounterName = "Bytes Sent";
bytesSentPerformanceCounter.InstanceName = str;
bytesSentPerformanceCounter.ReadOnly = false;
listBytesSentPerformanceCounter.Add(bytesSentPerformanceCounter);
bytesReceivedPerformanceCounter.CategoryName = ".NET CLR Networking 4.0.0.0";
bytesReceivedPerformanceCounter.CounterName = "Bytes Received";
bytesReceivedPerformanceCounter.InstanceName = str;
bytesReceivedPerformanceCounter.ReadOnly = false;
listBytesReceivedPerformanceCounter.Add(bytesReceivedPerformanceCounter);
}
}
//public float GetBytesSent()
//{
// float bytesSent = bytesSentPerformanceCounter.RawValue;
// return bytesSent;
//}
//public float GetBytesReceived()
//{
// float bytesReceived = bytesReceivedPerformanceCounter.RawValue;
// return bytesReceived;
//}
private static List<string> GetInstanceName()
{
// Used Reflector to find the correct formatting:
string assemblyName = GetAssemblyName();
if ((assemblyName == null) || (assemblyName.Length == 0))
{
assemblyName = AppDomain.CurrentDomain.FriendlyName;
}
StringBuilder builder = new StringBuilder(assemblyName);
for (int i = 0; i < builder.Length; i++)
{
switch (builder[i])
{
case '/':
case '\\':
case '#':
builder[i] = '_';
break;
case '(':
builder[i] = '[';
break;
case ')':
builder[i] = ']';
break;
}
}
List<string> listInstanceName = new List<string>();
Process[] listProcess = Process.GetProcesses();
string InstanceName;
foreach (Process pro in listProcess)
{
InstanceName = string.Format(CultureInfo.CurrentCulture,
"{0}[{1}]",
builder.ToString(),
pro.Id);
listInstanceName.Add(InstanceName);
}
return listInstanceName;
}
private static string GetAssemblyName()
{
string str = null;
Assembly entryAssembly = Assembly.GetEntryAssembly();
if (entryAssembly != null)
{
AssemblyName name = entryAssembly.GetName();
if (name != null)
{
str = name.Name;
}
}
return str;
}
public static void Main()
{
NetworkTraffic networkTraffic = new NetworkTraffic();
try
{
while (true)
{
WebRequest webRequest = WebRequest.Create("http://www.google.com");
webRequest.Method = "GET";
using (WebResponse response = webRequest.GetResponse())
using (Stream responseStream = response.GetResponseStream())
using (StreamReader reader = new StreamReader(responseStream))
{
}
//Console.WriteLine("Bytes sent: {0}", networkTraffic.GetBytesSent());
//Console.WriteLine("Bytes received: {0}", networkTraffic.GetBytesReceived());
foreach (PerformanceCounter send in networkTraffic.listBytesSentPerformanceCounter)
{
Console.WriteLine("Instance name: " + send.InstanceName + " Sent: " + send.RawValue);
}
Thread.Sleep(1000);
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
Console.ReadLine();
}
}
}
Anyone can help me resolve this.
Related to my other question except now I try async hoping it would fix the issues. It doesn't.
I'm trying to create a simple SOCKS5 server. I set my browser (firefox) to use this program as a SOCKS5. The idea is a program connects to the proxy server, give it information required and the server just simply reads/writes data from one connection to the other. This one simply does that and doesn't log nor filter anything. It is dead simple but because of the CPU issue and the fact it takes several seconds to connect to a site after you hit a few pages makes it completely unusable. How on earth is this eating up so much CPU? And why does it take a long time to connect to a site? Both async and sync suffer from this
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net.Sockets;
using System.Timers;
using System.IO;
using System.Net;
using System.Threading;
namespace ProxyTest
{
class Program
{
static ManualResetEvent tcpClientConnected =new ManualResetEvent(false);
static void Main(string[] args)
{
var s2 = new TcpListener(9998);
s2.Start();
Task.Run(() =>
{
while (true)
{
tcpClientConnected.Reset();
s2.BeginAcceptTcpClient(Blah, s2);
tcpClientConnected.WaitOne();
}
});
while (true)
System.Threading.Thread.Sleep(10000000);
}
static void Blah(IAsyncResult ar)
{
try
{
Console.WriteLine("Connection");
TcpListener listener = (TcpListener)ar.AsyncState;
using (var socketin = listener.EndAcceptTcpClient(ar))
{
tcpClientConnected.Set();
var ns1 = socketin.GetStream();
var r1 = new BinaryReader(ns1);
var w1 = new BinaryWriter(ns1);
if (false)
{
var s3 = new TcpClient();
s3.Connect("127.0.0.1", 9150);
var ns3 = s3.GetStream();
var r3 = new BinaryReader(ns3);
var w3 = new BinaryWriter(ns3);
while (true)
{
while (ns1.DataAvailable)
{
var b = ns1.ReadByte();
w3.Write((byte)b);
//Console.WriteLine("1: {0}", b);
}
while (ns3.DataAvailable)
{
var b = ns3.ReadByte();
w1.Write((byte)b);
Console.WriteLine("2: {0}", b);
}
}
}
{
if (!(r1.ReadByte() == 5 && r1.ReadByte() == 1))
return;
var c = r1.ReadByte();
for (int i = 0; i < c; ++i)
r1.ReadByte();
w1.Write((byte)5);
w1.Write((byte)0);
}
{
if (!(r1.ReadByte() == 5 && r1.ReadByte() == 1))
return;
if (r1.ReadByte() != 0)
return;
}
byte[] ipAddr = null;
string hostname = null;
var type = r1.ReadByte();
switch (type)
{
case 1:
ipAddr = r1.ReadBytes(4);
break;
case 3:
hostname = Encoding.ASCII.GetString(r1.ReadBytes(r1.ReadByte()));
break;
case 4:
throw new Exception();
}
var nhport = r1.ReadInt16();
var port = IPAddress.NetworkToHostOrder(nhport);
var socketout = new TcpClient();
if (hostname != null)
socketout.Connect(hostname, port);
else
socketout.Connect(new IPAddress(ipAddr), port);
w1.Write((byte)5);
w1.Write((byte)0);
w1.Write((byte)0);
w1.Write(type);
switch (type)
{
case 1:
w1.Write(ipAddr);
break;
case 2:
w1.Write((byte)hostname.Length);
w1.Write(Encoding.ASCII.GetBytes(hostname), 0, hostname.Length);
break;
}
w1.Write(nhport);
var buf1 = new byte[4096];
var buf2 = new byte[4096];
var ns2 = socketout.GetStream();
var r2 = new BinaryReader(ns2);
var w2 = new BinaryWriter(ns2);
Task.Run(() =>
{
var re = new ManualResetEvent(false);
while (true)
{
re.Reset();
ns1.BeginRead(buf1, 0, buf1.Length, ReadCallback, new A() { buf = buf1, thisSocket = socketin, otherSocket = socketout, thisStream = ns1, otherStream = ns2, re=re });
re.WaitOne();
}
});
Task.Run(() =>
{
var re = new ManualResetEvent(false);
while (true)
{
re.Reset();
ns2.BeginRead(buf2, 0, buf2.Length, ReadCallback, new A() { buf = buf2, thisSocket = socketout, otherSocket = socketin, thisStream = ns2, otherStream = ns1, re = re });
re.WaitOne();
}
});
while (true)
{
if (socketin.Connected == false)
return;
Thread.Sleep(100);
}
}
}
catch { }
}
class A { public byte[] buf; public TcpClient thisSocket, otherSocket; public NetworkStream thisStream, otherStream; public ManualResetEvent re;};
static void ReadCallback(IAsyncResult ar)
{
try
{
var a = (A)ar.AsyncState;
var ns1 = a.thisStream;
var len = ns1.EndRead(ar);
a.otherStream.Write(a.buf, 0, len);
a.re.Set();
}
catch
{
}
}
}
}
Caveat: I had to adjust things slightly since I'm not using 4.5.
Task.Run() --> new Thread().Start()
You are using far too many threads.
Simply attempting to load this question in stackoverflow caused 30+ threads to spawn, which reproduces the behavior seen using Task.Run().
With your code cut down to a single thread per connection, my CPU usage is hovering around 0%. Everything loads quickly.
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.ComponentModel;
using System.Data;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net.Sockets;
using System.Timers;
using System.IO;
using System.Net;
using System.Threading;
namespace SOCKS5
{
static class Program
{
static void Main()
{
var s2 = new TcpListener(9998);
s2.Start();
while (true)
{
if (s2.Pending())
{
Thread test = new Thread(() =>
{
using (TcpClient client = s2.AcceptTcpClient())
{
Blah(client);
}
});
test.Start();
}
Thread.Sleep(10);
}
}
static void Blah(TcpClient listener)
{
try
{
Console.WriteLine("Connection");
//TcpListener listener = (TcpListener)ar.AsyncState;
//tcpClientConnected.Set();
var ns1 = listener.GetStream();
var r1 = new BinaryReader(ns1);
var w1 = new BinaryWriter(ns1);
if (false)
{
var s3 = new TcpClient();
s3.Connect("127.0.0.1", 9150);
var ns3 = s3.GetStream();
var r3 = new BinaryReader(ns3);
var w3 = new BinaryWriter(ns3);
while (true)
{
while (ns1.DataAvailable)
{
var b = ns1.ReadByte();
w3.Write((byte)b);
//Console.WriteLine("1: {0}", b);
}
while (ns3.DataAvailable)
{
var b = ns3.ReadByte();
w1.Write((byte)b);
Console.WriteLine("2: {0}", b);
}
}
}
{
if (!(r1.ReadByte() == 5 && r1.ReadByte() == 1))
return;
var c = r1.ReadByte();
for (int i = 0; i < c; ++i)
r1.ReadByte();
w1.Write((byte)5);
w1.Write((byte)0);
}
{
if (!(r1.ReadByte() == 5 && r1.ReadByte() == 1))
return;
if (r1.ReadByte() != 0)
return;
}
byte[] ipAddr = null;
string hostname = null;
var type = r1.ReadByte();
switch (type)
{
case 1:
ipAddr = r1.ReadBytes(4);
break;
case 3:
hostname = Encoding.ASCII.GetString(r1.ReadBytes(r1.ReadByte()));
break;
case 4:
throw new Exception();
}
var nhport = r1.ReadInt16();
var port = IPAddress.NetworkToHostOrder(nhport);
var socketout = new TcpClient();
if (hostname != null)
socketout.Connect(hostname, port);
else
socketout.Connect(new IPAddress(ipAddr), port);
w1.Write((byte)5);
w1.Write((byte)0);
w1.Write((byte)0);
w1.Write(type);
switch (type)
{
case 1:
w1.Write(ipAddr);
break;
case 2:
w1.Write((byte)hostname.Length);
w1.Write(Encoding.ASCII.GetBytes(hostname), 0, hostname.Length);
break;
}
w1.Write(nhport);
var buf1 = new byte[4096];
var buf2 = new byte[4096];
var ns2 = socketout.GetStream();
DateTime last = DateTime.Now;
while ((DateTime.Now - last).TotalMinutes < 5.0)
{
if (ns1.DataAvailable)
{
int size = ns1.Read(buf1, 0, buf1.Length);
ns2.Write(buf1, 0, size);
last = DateTime.Now;
}
if (ns2.DataAvailable)
{
int size = ns2.Read(buf2, 0, buf2.Length);
ns1.Write(buf2, 0, size);
last = DateTime.Now;
}
Thread.Sleep(10);
}
}
catch { }
finally
{
try
{
listener.Close();
}
catch (Exception) { }
}
}
}
}
Edit:
This ended up being kinda fun to mess with.
After routing Firefox traffic through this for a few hours, some observations.
Never noticed a regular pattern to determine when to close connections. Letting threads terminate after they've been idle for 5 minutes (no rx/tx) keeps the thread count fairly low. It's a pretty safe bound that allows services such as gmail chat to keep functioning.
For some reason, the program would occasionally not receive requests from the browser, which would report a timeout. No notification of a missed request in the program, nothing. Only noticed when browsing stackoverflow. Still haven't figured that one out.
There are a few things going on here!
The async calls are all called synchronous style. As in, the thread that starts the operation calls a WaitOne - this basically just makes it equivalent to a synchonous call, no different.
Sleep loops are bad.
A sleep(1) loop will respond quickly but use some CPU, a sleep(1000) loop will respond slowly but use less CPU.
Having a dozen threads in a sleep loop doesn't use much CPU, but if the number of threads keeps increasing, CPU usage will become significant.
The best way is to use async calls instead of polling.
Lots of tasks running loops. Without guaranteed exit paths these cause the thread count to skyrocket.
If you are forwarding data from socket A to socket B, you need to act when either socket is closed: cease forwarding, ensure that pending writes complete and close the sockets.
The current implementation doesn't properly ensure both forwarding tasks are closed if one closes, and the technique of starting a task then blocking on a manual reset event can fail if the task gets an exception prior to setting the event. Both cases leave a task running ad infinitum.
Checking Socket.Connected seems like an obvious thing to do but in practice this is just a cache of the whether the last IO operation encoundered a disconnect.
I prefer to act on "zero recv"s which are your first notification of a disconnect.
I knocked up a quick async version of your original synchronous routine using PowerThreading via NuGet (this is a way of doing async routines prior to framework 4.5).
This works using TcpListener with zero cpu usage and very low number of threads.
This can be done in vanilla c# using async/await... I just don't know how yet :)
using System;
using System.Collections.Generic;
using System.Text;
namespace AeProxy
{
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Threading;
// Need to install Wintellect.Threading via NuGet for this:
using Wintellect.Threading.AsyncProgModel;
class Program
{
static void Main(string[] args)
{
var ae = new AsyncEnumerator() {SyncContext = null};
var mainOp = ae.BeginExecute(ListenerFiber(ae), null, null);
// block until main server is finished
ae.EndExecute(mainOp);
}
static IEnumerator<int> ListenerFiber(AsyncEnumerator ae)
{
var listeningServer = new TcpListener(IPAddress.Loopback, 9998);
listeningServer.Start();
while (!ae.IsCanceled())
{
listeningServer.BeginAcceptTcpClient(ae.End(0, listeningServer.EndAcceptTcpClient), null);
yield return 1;
if (ae.IsCanceled()) yield break;
var clientSocket = listeningServer.EndAcceptTcpClient(ae.DequeueAsyncResult());
var clientAe = new AsyncEnumerator() { SyncContext = null };
clientAe.BeginExecute(
ClientFiber(clientAe, clientSocket),
ar =>
{
try
{
clientAe.EndExecute(ar);
}
catch { }
}, null);
}
}
static long clients = 0;
static IEnumerator<int> ClientFiber(AsyncEnumerator ae, TcpClient clientSocket)
{
Console.WriteLine("ClientFibers ++{0}", Interlocked.Increment(ref clients));
try
{
// original code to do handshaking and connect to remote host
var ns1 = clientSocket.GetStream();
var r1 = new BinaryReader(ns1);
var w1 = new BinaryWriter(ns1);
if (!(r1.ReadByte() == 5 && r1.ReadByte() == 1)) yield break;
var c = r1.ReadByte();
for (int i = 0; i < c; ++i) r1.ReadByte();
w1.Write((byte)5);
w1.Write((byte)0);
if (!(r1.ReadByte() == 5 && r1.ReadByte() == 1)) yield break;
if (r1.ReadByte() != 0) yield break;
byte[] ipAddr = null;
string hostname = null;
var type = r1.ReadByte();
switch (type)
{
case 1:
ipAddr = r1.ReadBytes(4);
break;
case 3:
hostname = Encoding.ASCII.GetString(r1.ReadBytes(r1.ReadByte()));
break;
case 4:
throw new Exception();
}
var nhport = r1.ReadInt16();
var port = IPAddress.NetworkToHostOrder(nhport);
var socketout = new TcpClient();
if (hostname != null) socketout.Connect(hostname, port);
else socketout.Connect(new IPAddress(ipAddr), port);
w1.Write((byte)5);
w1.Write((byte)0);
w1.Write((byte)0);
w1.Write(type);
switch (type)
{
case 1:
w1.Write(ipAddr);
break;
case 3:
w1.Write((byte)hostname.Length);
w1.Write(Encoding.ASCII.GetBytes(hostname), 0, hostname.Length);
break;
}
w1.Write(nhport);
using (var ns2 = socketout.GetStream())
{
var forwardAe = new AsyncEnumerator() { SyncContext = null };
forwardAe.BeginExecute(
ForwardingFiber(forwardAe, ns1, ns2), ae.EndVoid(0, forwardAe.EndExecute), null);
yield return 1;
if (ae.IsCanceled()) yield break;
forwardAe.EndExecute(ae.DequeueAsyncResult());
}
}
finally
{
Console.WriteLine("ClientFibers --{0}", Interlocked.Decrement(ref clients));
}
}
private enum Operation { OutboundWrite, OutboundRead, InboundRead, InboundWrite }
const int bufsize = 4096;
static IEnumerator<int> ForwardingFiber(AsyncEnumerator ae, NetworkStream inputStream, NetworkStream outputStream)
{
while (!ae.IsCanceled())
{
byte[] outputRead = new byte[bufsize], outputWrite = new byte[bufsize];
byte[] inputRead = new byte[bufsize], inputWrite = new byte[bufsize];
// start off output and input reads.
// NB ObjectDisposedExceptions can be raised here when a socket is closed while an async read is in progress.
outputStream.BeginRead(outputRead, 0, bufsize, ae.End(1, ar => outputStream.EndRead(ar)), Operation.OutboundRead);
inputStream.BeginRead(inputRead, 0, bufsize, ae.End(1, ar => inputStream.EndRead(ar)), Operation.InboundRead);
var pendingops = 2;
while (!ae.IsCanceled())
{
// wait for the next operation to complete, the state object passed to each async
// call can be used to find out what completed.
if (pendingops == 0) yield break;
yield return 1;
if (!ae.IsCanceled())
{
int byteCount;
var latestEvent = ae.DequeueAsyncResult();
var currentOp = (Operation)latestEvent.AsyncState;
if (currentOp == Operation.InboundRead)
{
byteCount = inputStream.EndRead(latestEvent);
if (byteCount == 0)
{
pendingops--;
outputStream.Close();
continue;
}
Array.Copy(inputRead, outputWrite, byteCount);
outputStream.BeginWrite(outputWrite, 0, byteCount, ae.EndVoid(1, outputStream.EndWrite), Operation.OutboundWrite);
inputStream.BeginRead(inputRead, 0, bufsize, ae.End(1, ar => inputStream.EndRead(ar)), Operation.InboundRead);
}
else if (currentOp == Operation.OutboundRead)
{
byteCount = outputStream.EndRead(latestEvent);
if (byteCount == 0)
{
pendingops--;
inputStream.Close();
continue;
}
Array.Copy(outputRead, inputWrite, byteCount);
inputStream.BeginWrite(inputWrite, 0, byteCount, ae.EndVoid(1, inputStream.EndWrite), Operation.InboundWrite);
outputStream.BeginRead(outputRead, 0, bufsize, ae.End(1, ar => outputStream.EndRead(ar)), Operation.OutboundRead);
}
else if (currentOp == Operation.InboundWrite)
{
inputStream.EndWrite(latestEvent);
}
else if (currentOp == Operation.OutboundWrite)
{
outputStream.EndWrite(latestEvent);
}
}
}
}
}
}
}
In this line...
while (true)
System.Threading.Thread.Sleep(10000000);
Will not be better to replace it by a simple:
Console.ReadKey();
is the only CPU consumption thing I see.
Also, as a suggestion, you should limit the number of incoming connections and use a Thread pool pattern (in a queue or something).
You should take a look at Overlapped I/O.
One Thread per connection maybe works fine but in general it's bad.
You should use async versions of TcpClient methods instead of spawning threads.