I need to access data from two different DbContexts at the same time, making sure each uses READ UNCOMMITTED for their queries (really, the important thing is that it doesn't lock the rows it iterates over - adding WITH NO LOCK to the query would work too).
How can you do this using Entity Framework? If I wrap each of the two queries in a TransactionScope, it tries to promote the transaction to MSDTC which isn't an option for us.
private static IEnumerable<Image> EnumerateSourceImages()
{
using (var dbContext = new SourceDbContext())
{
using (var transScope = new TransactionScope(
TransactionScopeOption.RequiresNew,
new TransactionOptions() {
IsolationLevel = IsolationLevel.ReadUncommitted
}
)
)
{
var imagesSourceQuery = dbContext.ImageDatas
.AsNoTracking()
.OrderBy(imageData => imageData.ImageID)
foreach (var image in imagesSourceQuery)
{
yield return image;
}
transScope.Complete();
}
}
}
private static IEnumerable<Image> EnumerateDestinationImages()
{
using (var dbContext = new DestinationDbContext())
{
using (var transScope = new TransactionScope(
TransactionScopeOption.RequiresNew,
new TransactionOptions() {
IsolationLevel = IsolationLevel.ReadUncommitted
}
)
)
{
var imagesSourceQuery = dbContext.ImageDatas
.AsNoTracking()
.OrderBy(imageData => imageData.ImageID)
foreach (var image in imagesSourceQuery)
{
yield return image;
}
transScope.Complete();
}
}
}
private static void main(string[] args){
{
IEnumerator<ItemImage> sourceImagesEnumerator = null;
IEnumerator<ItemImage> destImagesEnumerator = null;
try{
sourceImagesEnumerator = EnumerateSourceImages().GetEnumerator();
destImagesEnumerator = EnumerateDestinationImages().GetEnumerator();
bool sourceHasMore = sourceImagesEnumerator.MoveNext();
//Exception on next line about MSDTC Promotion
bool destHasMore = destImagesEnumerator.MoveNext();
} finally{
if(sourceImagesEnumerator != null) sourceImagesEnumerator.Dispose();
if(destImagesEnumerator != null) destImagesEnumerator.Dispose():
}
}
Did you try setting enlist=false in your connection string?
http://forums.asp.net/t/1401606.aspx/1
Brgrds,
Lari
Related
I try to code a MongoDB change stream. But if I do some changes, my code doesn't detect the changes. If the code detects the changes I have to get the operation type - according to my understanding. Moreover, how can I insert the changes in the other database? Is it so "easy", that I code a new database connection and update/insert/delete the collection/database?
What did I Code?
`
lang-cs
string mongo_db_client = "mongodb://localhost:27017";
string m_database = "myDataBase";
string m_collection ="myCollection";
MongoClient dbClient = new MongoClient(mongo_db_client);
var database = dbClient.GetDatabase(m_database);
var collection = database.GetCollection<BsonDocument>(m_collection);
var cursor = collection.Watch();
var the_operation_type = null;
if(cursor != null)
{
using(cursor)
{
var curr = cursor.Current;
if (curr != null)
{
foreach (var change in curr)
{
try
{
the_operation_type= change.OperationType;
}
catch(NullReferenceException e)
{
Log(e.ToString());
}
}
}
}
}
`
I try to insert a big load of data to my SQL Server with out reaching the maximum connection.
I already tried to max the pool size in my connection string like this:
optionsBuilder.UseSqlServer(#"Server=(localdb)\mssqllocaldb;Database=Echo;Max Pool Size=5000;Pooling=True;");
Is it possible to use one connection for each thread?
Here is my code
public static void GenerateMeridien()
{
List<Parallele> paralleleList = null;
using (var context = new EchoContext())
{
paralleleList = context.Paralleles.ToList();
}
foreach (Parallele parallele in paralleleList)
{
ThreadPool.QueueUserWorkItem(new WaitCallback(MeridienTaskCallback), parallele);
}
}
private static void MeridienTaskCallback(object paralleleObj)
{
Parallele parallele = (Parallele)paralleleObj;
// The context always reaches maximum connection because the garbage collector is not fast enough
using (var context = new EchoContext())
{
List<Square> squareList = SquareLogic.GenerateSquare(parallele);
context.Squares.AddRange(squareList);
context.SaveChanges();
Console.Write("*");
}
}
Option A. Do work in parallel, save once
Do you have to save after each calculation? Can you do calculations in parallel and save after?
List<Parallele> paralleleList = null;
using (var context = new EchoContext())
{
paralleleList = context.Paralleles.ToList();
}
List<Square> squares = DoAllWorkInParallel(paralleleList);
// Save once:
using (var context = new EchoContext())
{
context.Squares.AddRange(squares);
context.SaveChanges();
}
Option B. Save all the time
I think you should use Parallel.ForEach with MaxDegreeOfParallelism or use Task.WhenAll.
Parallel.ForEach
For example:
public static async Task GenerateMeridien()
{
List<Parallele> paralleleList = null;
using (var context = new EchoContext())
{
paralleleList = context.Paralleles.ToList();
}
Parallel.ForEach(
source: paralleleList,
parallelOptions: new ParallelOptions{MaxDegreeOfParallelism = x},
body: parallele => Work(parallele));
}
private static void Work(Parallele parallele)
{
using (var context = new EchoContext())
{
List<Square> squareList = SquareLogic.GenerateSquare(parallele);
context.Squares.AddRange(squareList);
await context.SaveChanges();
}
}
Task.WhenAll
public static async Task GenerateMeridien()
{
List<Parallele> paralleleList = null;
using (var context = new EchoContext())
{
paralleleList = context.Paralleles.ToList();
}
var tasks = new List<Task>();
foreach (Parallele parallele in paralleleList)
{
tasks.Add(WorkAsync(parallele));
}
await Task.WhenAll(tasks.ToArray());
}
private static async Task WorkAsync(Parallele parallele)
{
using (var context = new EchoContext())
{
List<Square> squareList = SquareLogic.GenerateSquare(parallele);
context.Squares.AddRange(squareList);
await context.SaveChangesAsync();
}
}
I'm struggling to find an answer to a problem I've been experiencing with C#'s popular Postgres library, Npgsql. I'm not sure if it's wholly a problem with Npgsql or not, though I suspect it is because my code is very straight forward. The issue I'm seeing is: when I call an async method on Npgsql sometimes, not all the time, the thread becomes locked. This issue occurs randomly from what I can tell. The result, being that I'm running in a Microsoft Orleans environment (which may be relevant to finding the solution), is that the thread locks indefinitely, thus making one of Orleans' worker threads unable to process work. As I make more Npgsql calls, these locked threads stack up and eventually the Orleans system is choked by thread exhaustion.
So I'm really at a loss for what the problem could be, but because the locking always happens in the same method, and because it appears to be happening in some subroutine of Npgsql, I think it's fair to investigate Npgsql further.
Here's my code which is used in an Orleans storage provider (special class which handles the system's persistence layer.)
var sql = $"SELECT * FROM objects WHERE id = #id";
using (var connection = new NpgsqlConnection(connectionString))
using (var cmd = new NpgsqlCommand(sql, connection))
{
try
{
await connection.OpenAsync();
cmd.Parameters.AddWithValue("id", id);
using(var reader = await connection.ExecuteReaderAsync(cmd))
{
if (reader.HasRows)
{
var objects = await ProtobufSQL.DataReaderToType(modelType, reader);
var data = objects[0];
state.Data = data;
}
}
catch (Exception e)
{
Log.Error(1, e.Message, e);
}
}
This is the source for the ProtobufSQL class:
public class ProtobufSQL
{
public static List<Tuple<string, object>> FlattenToSQLColumns(IMessage message, MessageDescriptor descriptor, string prefix = null)
{
var fields = descriptor.Fields.InDeclarationOrder();
var columns = new List<Tuple<string, object>>();
for (var i = 0; i < fields.Count; i++)
{
var field = fields[i];
var columnName = field.Name.ToLower();
if (field.Name == "id")
{
ByteString bytes = (ByteString)field.Accessor.GetValue(message);
var uuid = new Guid(bytes.ToByteArray());
columns.Add(new Tuple<string, object>("id", uuid));
}
else if (field.FieldType == FieldType.Message)
{
var embeddedDescriptor = field.MessageType;
var embeddedMessage = field.Accessor.GetValue(message);
if (field.IsRepeated)
{
throw new Exception("Repeated complex types are not supported, create a foreign key reference in a new object instead.");
}
else
{
columns.AddRange(FlattenToSQLColumns((IMessage)embeddedMessage, embeddedDescriptor, $"{columnName}."));
}
}
else if (field.FieldType == FieldType.Group)
{
throw new Exception("Groups are not supported by ProtobufSQL.");
}
else
{
var columnValue = field.Accessor.GetValue(message);
var key = prefix + columnName;
if (field.IsRepeated)
{
var enumerableColumnValue = columnValue as IEnumerable;
Type listTypeOf = enumerableColumnValue.GetType().GetGenericArguments()[0];
Type listType = typeof(List<>).MakeGenericType(listTypeOf);
dynamic valueList = Activator.CreateInstance(listType);
foreach (var item in enumerableColumnValue)
{
valueList.Add((dynamic)item);
}
columns.Add(new Tuple<string, object>(key, valueList.ToArray()));
}
else
{
columns.Add(new Tuple<string, object>(key, columnValue));
}
}
}
return columns;
}
public static async Task<IMessage[]> DataReaderToType(Type type, DbDataReader reader)
{
var descriptor = (MessageDescriptor)type.GetProperty("Descriptor").GetValue(null);
IList<IMessage> objects = new List<IMessage>();
while (await reader.ReadAsync())
{
var obj = Activator.CreateInstance(type);
TraverseDbRow(reader, descriptor, obj);
objects.Add((IMessage)obj);
}
return objects.ToArray();
}
private static void TraverseDbRow(DbDataReader reader, MessageDescriptor descriptor, object obj, string prefix = null)
{
var fields = descriptor.Fields.InFieldNumberOrder();
for (var i = 0; i < fields.Count; i++)
{
var field = fields[i];
if (field.FieldType == FieldType.Message)
{
if (field.IsRepeated)
{
// Repeated embedded types will be broken out into a separate table,
// so there's no need to handle them here.
}
else if (field.IsMap)
{
throw new Exception("Maps are not yet supported by ProtobufSQL.");
}
else
{
var embeddedDescriptor = field.MessageType;
var embeddedObj = Activator.CreateInstance(embeddedDescriptor.ClrType);
TraverseDbRow(reader, embeddedDescriptor, embeddedObj, $"{prefix}{field.Name}.");
}
}
else if (field.FieldType == FieldType.Group)
{
throw new Exception("Groups are not supported by ProtobufSQL.");
}
else
{
var columnName = prefix + field.Name;
try
{
var columnValue = reader[columnName];
var propertyInfo = obj.GetType().GetProperty(field.Name, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
if (field.Name == "id")
{
var guid = (Guid)columnValue;
ByteString bytes = ByteString.CopyFrom(guid.ToByteArray());
propertyInfo.SetValue(obj, bytes);
}
else if (field.IsRepeated)
{
var repeated = propertyInfo.GetValue(obj);
var addRange = repeated.GetType().GetMethod("AddRange");
addRange.Invoke(repeated, new object[] { columnValue });
}
else if (field.IsMap)
{
throw new Exception("Maps are not yet supported by ProtobufSQL.");
}
else
{
propertyInfo.SetValue(obj, Convert.ChangeType(columnValue, propertyInfo.PropertyType));
}
}
catch (IndexOutOfRangeException e)
{
// columnName was not present in the response
}
}
}
}
}
I also have this screenshot of the thread's stack when it got locked:
I'm really unsure what to make of all this. Hopefully someone is out there with a bit of knowledge that can help me proceed! Thanks.
Here is my code
public static string UpdateEmptyCaseRevierSet() {
string response = string.Empty;
using (System.Transactions.TransactionScope tran = new System.Transactions.TransactionScope()) {
using (var db = new Entities.WaveEntities()) {
var maxCaseReviewersSetID = db.CaseReviewerSets.Select(crs => crs.CaseReviewersSetId).Max();
var emptyCHList = db.CaseHistories.Where(ch => ch.CaseReviewersSetID == null && ch.IsLatest == true && ch.StatusID != 100).ToList();
for(int i=0; i < emptyCHList.Count; i++) {
var emptyCH = emptyCHList[i];
var newCaseReviewerSET = new Entities.CaseReviewerSet();
newCaseReviewerSET.CreationCHID = emptyCH.CHID;
db.CaseReviewerSets.Add(newCaseReviewerSET);
emptyCH.CaseReviewerSet = newCaseReviewerSET;
}
db.SaveChanges();
}
tran.Complete();
}
return response;
}
The exception occures on "db.SaveChanges()"
I saw in another post with the same error message something about "it seems I cannot have two connections opened to the same database with the TransactionScope block." but I dont think that this has anything to do with my case.
Additionally the number of records to insert and update in total are 2700, witch is not that many really. But it does take quite a lot of time to complete the for statement (10 minutes or so). Since everything happening within the for statement is actually happening in the memory can someone please explane why is this taking so long ?
You can try as shown below using latest db.Database.BeginTransaction API.
Note : use foreach instead of for
using (var db = new Entities.WaveEntities())
{
using (var dbContextTransaction = db.Database.BeginTransaction())
{
try
{
var maxCaseReviewersSetID = db.CaseReviewerSets.Select(crs => crs.CaseReviewersSetId).Max();
var emptyCHList = db.CaseHistories.Where(ch => ch.CaseReviewersSetID == null && ch.IsLatest == true && ch.StatusID != 100).ToList();
foreach(var ch in emptyCHList) {
var newCaseReviewerSET = new Entities.CaseReviewerSet();
newCaseReviewerSET.CreationCHID = ch.CHID;
db.CaseReviewerSets.Add(newCaseReviewerSET);
}
db.SaveChanges();
dbContextTransaction.Commit();
}
catch (Exception)
{
dbContextTransaction.Rollback();
}
}
}
How can I changedb(redis command select), when I used the pool of redis.
I want write host and read host from different DB.
for example:
just now I only have one redis server,so the readWriteHosts = readOnlyHosts
pool = RedisDao.CreateManager(hostIp, hostIp);
public static PooledRedisClientManager CreateManager(string[] readWriteHosts, string[] readOnlyHosts)
{
return new PooledRedisClientManager(readWriteHosts, readOnlyHosts, new RedisClientManagerConfig
{
MaxWritePoolSize = 50,//
MaxReadPoolSize = 5,//
AutoStart = true,
});
}
public RedisDB ReadRedisForModel(String ID)
{
//here I want to use DB number is day%15
using (var redis = pool.GetClient())
{
RedisDB model = new RedisDB();
Dictionary<string, string> dic = redis.GetAllEntriesFromHash(keyPrefix + ID);
model.ID = ID;//Int32.Parse(ids[i]);
return model;
}
}
public void WriteRedis(RedisDB model)
{
//here I want to use DB number is (day-1)%15
using (var redis = pool.GetClient())
{
EDIT:
I find a way to set different DB,But I feel this solution is not best way.
if(redis is RedisClient)
{
long db = redis.DB;//db always = 0;
((RedisClient)redis).ChangeDB((day-1)%15);
}
Is it need to lock thread? when i am read or write to redis.
I am afraid, I got the same redis client in mutil-thread . then the redis DB is ?
Edit end
int time = DateTimeUtil.ConvertDateTimeInt(DateTime.Now);
model.ID = time + redis.Increment(incrementKey, 1) + "";//.Incr("ID");
using (var pip = redis.CreatePipeline())
{
pip.QueueCommand(r => r.AddItemToList(primaryKey, model.ID + ""));
pip.Flush();
};
};
}
I got redisClient from pool, but redisClient is not have the function of changeDB.
So anybody kowns how to set it?
for example:
//write
bool IsNeedChangeDB=true;
int WriteDBNumber=3
public static PooledRedisClientManager pool = RedisDao.CreateManager(hostIp, hostIp);
using (var redis = pool.GetClient())
{
if (redis is RedisClient && IsNeedChangeDB)
{
if (redis.Db != this.WriteDBNumber)
{
((RedisClient)redis).ChangeDb(this.WriteDBNumber);
}
else
{
Trace.WriteLine("it is a test" + redis.Host);
}
}
redis.Set<string>("key","value");
}
int ReadDBNumber=3;
//read
protected IRedisClient GetRedisClient()
{
var redis = pool.GetClient();
if (redis is RedisClient && IsNeedChangeDB)
{
if (redis.Db != this.ReadDBNumber)
((RedisClient)redis).ChangeDb(this.ReadDBNumber);
}
return redis;
}