I have a function which grabs the data from API and stores into the database.
public async Task GetApps()
{
var request = new HttpRequestMessage(HttpMethod.Get, "apps?per_page=200");
var response = await _client_SB.SendAsync(request);
var json = await response.Content.ReadAsStringAsync();
AppsClass.AppsRootobject model = JsonConvert.DeserializeObject<AppsClass.AppsRootobject>(json);
using (var scope = _scopeFactory.CreateScope())
{
var _DBcontext = scope.ServiceProvider.GetRequiredService<PCFStatusContexts>();
foreach (var item in model.resources)
{
var g = Guid.Parse(item.guid);
var x = _DBcontext.Apps.FirstOrDefault(o => o.AppGuid == g);
if (x == null)
{
_DBcontext.Apps.Add(new Apps
{
AppGuid = Guid.Parse(item.guid),
Name = item.name,
State = item.state,
CreatedAt = item.created_at,
UpdatedAt = item.updated_at,
SpaceGuid = Guid.Parse(item.relationships.space.data.guid),
Foundation = 2,
Timestamp = DateTime.Now
});
}
else if (x.UpdatedAt != item.updated_at)
{
x.State = item.state;
x.CreatedAt = item.created_at;
x.UpdatedAt = item.updated_at;
x.Timestamp = DateTime.Now;
}
var guids = model.resources.Select(r => Guid.Parse(r.guid));
var apps = _DBcontext.Apps.Where(o => guids.Contains(o.AppGuid) == false && o.Foundation == 2);
foreach (var app in apps)
{
app.DeletedAt = DateTime.Now;
}
}
await _DBcontext.SaveChangesAsync();
}
}
The above function works just fine except the snippet for DeletedAt
var guids = model.resources.Select(r => Guid.Parse(r.guid));
var apps = _DBcontext.Apps.Where(o => guids.Contains(o.AppGuid) == false && o.Foundation == 2);
foreach (var app in apps)
{
app.DeletedAt = DateTime.Now;
}
it's not like it doesn't work at all but when the GetApps function called every second because of this snippet I'm getting the following error
An exception occurred while iterating over the results of a query for
context type 'TestApp.Context.DBContexts'.
System.InvalidOperationException: Timeout expired. The timeout period elapsed prior to obtaining a connection from the pool. This
may have occurred because all pooled connections were in use and max
pool size was reached.
If I remove DeletedAt snippet it works just fine but I need that functionality, is there any way to optimize this? can I make it async?
Related
I want to improve the performance and remove the delay in showing the data to the user on the screen. As per requirement, I need to get the list of the data from a different source, then get the further data from other sources based on the previous data which takes a lot of time and feel that executing them sequentially.
I am looking for the suggestion to improve the performance, asynchronously call the client and wait at the end and reduce the wait time of the request.
foreach (var n in player.data)
{
var request1 = new HttpRequestMessage(HttpMethod.Get, "https://api.*****.com/buckets/" + **** + "/tests/" + n.id);
var client1 = new HttpClient();
request1.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "****-b23a-*****-b1be-********");
HttpResponseMessage response1 = await client1.SendAsync(request1, HttpCompletionOption.ResponseHeadersRead);
List<dataroot> root1 = new List<dataroot>();
if (response1.StatusCode == System.Net.HttpStatusCode.OK)
{
try
{
var apiString1 = await response1.Content.ReadAsStringAsync();
var player1 = Newtonsoft.Json.JsonConvert.DeserializeObject<envRoot>(apiString1);
if (!string.IsNullOrEmpty(player1.data.environments[0].parent_environment_id))
{
player.data.Where(x => x.id == player1.data.environments[0].test_id).ToList().ForEach(s => s.isShared = true);
player.data.Where(x => x.id == player1.data.environments[0].test_id).ToList().ForEach(s => s.sharedEnvironmentId = player1.data.environments[0].parent_environment_id);
//player.data.Where(x=>x.id==player1.data.environments[0].test_id).ToList().ForEach(s=>s.sharedEnvironmentId=player1.data.environments[0].test_id);
}
player.data.Where(x => x.id == player1.data.environments[0].test_id).ToList().ForEach(s => s.normalenvironmentId = player1.data.environments[0].id);
}
catch (Exception ex)
{
var test = ex;
}
}
}
You can try the way I did in my sample below:
https://github.com/rajabb/RunningLongRunningTasksEfficientlyAndWaitAtEnd
The main part of code is:
List<Task> tasks = new List<Task>();
for (int i = 0; i < 100; i++)
{
tasks.Add(LongRunningTask.RunAsync(i.ToString()));
}
await Task.WhenAll(tasks.ToArray());
I have this function that updates a CouchDB Database but I want it to try updating again if the Response Code is conflict, i want it to have 3 tries, how do I do that?
public async Task<HttpResponseMessage> UpdateRecord(Profile latestProfile)
{
ProfileRecordByUpn profileRecord = await this.GetProfileByUpn(latestProfile);
Profile oldProfile = profileRecord.Rows.First().Value;
var client = this.clientFactory.CreateClient(NamedHttpClients.COUCHDB);
var formatter = new JsonMediaTypeFormatter();
formatter.SerializerSettings = new JsonSerializerSettings
{
Formatting = Formatting.Indented,
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
var query = HttpUtility.ParseQueryString(string.Empty);
query["rev"] = oldProfile.Rev;
//Setting the profile Active = true, because as of now we don't have any UI for disabling the account
latestProfile.Active = oldProfile.Active;
DateTimeOffset now = DateTimeOffset.Now;
latestProfile.Created = oldProfile.Created;
latestProfile.Modified = now;
//This will check if we the InApp boolean value changed then will set date to Enabled/Disabled
if (oldProfile.InApp != latestProfile.InApp)
{
if (latestProfile.InApp == true)
{
latestProfile.InAppEnabled = now;
latestProfile.InAppDisabled = oldProfile.InAppDisabled;
}
else
{
latestProfile.InAppDisabled = now;
latestProfile.InAppEnabled = oldProfile.InAppEnabled;
}
}
else
{
latestProfile.InAppEnabled = oldProfile.InAppEnabled;
latestProfile.InAppDisabled = oldProfile.InAppDisabled;
}
//This will check if we the SMS boolean value changed then will set date to Enabled/Disabled
if (oldProfile.SMS != latestProfile.SMS)
{
if (latestProfile.SMS == true)
{
latestProfile.SMSEnabled = now;
latestProfile.SMSDisabled = oldProfile.SMSDisabled;
}
else
{
latestProfile.SMSDisabled = now;
latestProfile.SMSEnabled = oldProfile.SMSEnabled;
}
}
else
{
latestProfile.SMSEnabled = oldProfile.SMSEnabled;
latestProfile.SMSDisabled = oldProfile.SMSDisabled;
}
//This will check if we the SMS boolean value changed then will set date to Enabled/Disabled
if (oldProfile.Email != latestProfile.Email)
{
if (latestProfile.Email == true)
{
latestProfile.EmailEnabled = now;
latestProfile.EmailDisabled = oldProfile.EmailDisabled;
}
else
{
latestProfile.EmailDisabled = now;
latestProfile.EmailEnabled = oldProfile.EmailEnabled;
}
}
else
{
latestProfile.EmailEnabled = oldProfile.EmailEnabled;
latestProfile.EmailDisabled = oldProfile.EmailDisabled;
}
var response = await this.couchDbClient.AuthenticatedQuery(async (c) => {
return await c.PutAsync($"{API_PROFILES_DB}/{oldProfile.Id.ToString()}?{query}", latestProfile, formatter);
}, NamedHttpClients.COUCHDB, client);
return response;
}
so I will be calling this function from another function? Do I make an another function which is a higher order function and pass this function as a parameter to that higher order function?
Higher-order functions in C# are implemented by methods taking delegates as parameters, usually an Action or Func delegate.
In this case, you should use an established library like Polly.
var policy = Policy
.HandleResult<HttpResponseMessage>(r => r.StatusCode == HttpStatusCode.Conflict)
.RetryAsync(3);
var result = await policy.ExecuteAsync(() => UpdateRecord(latestProfile));
Update to do it yourself (uncompiled and untested code):
async Task<HttpResponseMessage> MyRetry(Func<Task<HttpResponseMessage>> action)
{
for (int retries = 0; retries < 3; ++retries)
{
var result = await action();
if (result.StatusCode != HttpStatusCode.Conflict)
return result;
}
return await action();
}
The above code will retry 3 times, for 4 total calls if it keeps returning Conflict.
I have an SSIS package that's launching another SSIS package in a Foreach container; because the container reports completion as soon as it launched all the packages it had to launch, I need a way to make it wait until all "child" packages have completed.
So I implemented a little sleep-wait loop that basically pulls the Execution objects off the SSISDB for the ID's I'm interested in.
The problem I'm facing, is that a grand total of 0 Dts.Events.FireProgress events get fired, and if I uncomment the Dts.Events.FireInformation call in the do loop, then every second I get a message reported saying 23 packages are still running... except if I check in SSISDB's Active Operations window I see that most have completed already and 3 or 4 are actually running.
What am I doing wrong, why wouldn't runningCount contain the number of actually running executions?
using ssis = Microsoft.SqlServer.Management.IntegrationServices;
public void Main()
{
const string serverName = "REDACTED";
const string catalogName = "SSISDB";
var ssisConnectionString = $"Data Source={serverName};Initial Catalog=msdb;Integrated Security=SSPI;";
var ids = GetExecutionIDs(serverName);
var idCount = ids.Count();
var previousCount = -1;
var iterations = 0;
try
{
var fireAgain = true;
const int secondsToSleep = 1;
var sleepTime = TimeSpan.FromSeconds(secondsToSleep);
var maxIterations = TimeSpan.FromHours(1).TotalSeconds / sleepTime.TotalSeconds;
IDictionary<long, ssis.Operation.ServerOperationStatus> catalogExecutions;
using (var connection = new SqlConnection(ssisConnectionString))
{
var server = new ssis.IntegrationServices(connection);
var catalog = server.Catalogs[catalogName];
do
{
catalogExecutions = catalog.Executions
.Where(execution => ids.Contains(execution.Id))
.ToDictionary(execution => execution.Id, execution => execution.Status);
var runningCount = catalogExecutions.Count(kvp => kvp.Value == ssis.Operation.ServerOperationStatus.Running);
System.Threading.Thread.Sleep(sleepTime);
//Dts.Events.FireInformation(0, "ScriptMain", $"{runningCount} packages still running.", string.Empty, 0, ref fireAgain);
if (runningCount != previousCount)
{
previousCount = runningCount;
decimal completed = idCount - runningCount;
decimal percentCompleted = completed / idCount;
Dts.Events.FireProgress($"Waiting... {completed}/{idCount} completed", Convert.ToInt32(100 * percentCompleted), 0, 0, "", ref fireAgain);
}
iterations++;
if (iterations >= maxIterations)
{
Dts.Events.FireWarning(0, "ScriptMain", $"Timeout expired, requesting cancellation.", string.Empty, 0);
Dts.Events.FireQueryCancel();
Dts.TaskResult = (int)Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Canceled;
return;
}
}
while (catalogExecutions.Any(kvp => kvp.Value == ssis.Operation.ServerOperationStatus.Running));
}
}
catch (Exception exception)
{
if (exception.InnerException != null)
{
Dts.Events.FireError(0, "ScriptMain", exception.InnerException.ToString(), string.Empty, 0);
}
Dts.Events.FireError(0, "ScriptMain", exception.ToString(), string.Empty, 0);
Dts.Log(exception.ToString(), 0, new byte[0]);
Dts.TaskResult = (int)ScriptResults.Failure;
return;
}
Dts.TaskResult = (int)ScriptResults.Success;
}
The GetExecutionIDs function simply returns all execution ID's for the child packages, from my metadata database.
The problem is that you're re-using the same connection at every iteration. Turn this:
using (var connection = new SqlConnection(ssisConnectionString))
{
var server = new ssis.IntegrationServices(connection);
var catalog = server.Catalogs[catalogName];
do
{
catalogExecutions = catalog.Executions
.Where(execution => ids.Contains(execution.Id))
.ToDictionary(execution => execution.Id, execution => execution.Status);
Into this:
do
{
using (var connection = new SqlConnection(ssisConnectionString))
{
var server = new ssis.IntegrationServices(connection);
var catalog = server.Catalogs[catalogName];
catalogExecutions = catalog.Executions
.Where(execution => ids.Contains(execution.Id))
.ToDictionary(execution => execution.Id, execution => execution.Status);
}
And you'll get correct execution status every time. Not sure why the connection can't be reused, but keeping connections as short-lived as possible is always a good idea - and that's another proof.
Here is my code
public static string UpdateEmptyCaseRevierSet() {
string response = string.Empty;
using (System.Transactions.TransactionScope tran = new System.Transactions.TransactionScope()) {
using (var db = new Entities.WaveEntities()) {
var maxCaseReviewersSetID = db.CaseReviewerSets.Select(crs => crs.CaseReviewersSetId).Max();
var emptyCHList = db.CaseHistories.Where(ch => ch.CaseReviewersSetID == null && ch.IsLatest == true && ch.StatusID != 100).ToList();
for(int i=0; i < emptyCHList.Count; i++) {
var emptyCH = emptyCHList[i];
var newCaseReviewerSET = new Entities.CaseReviewerSet();
newCaseReviewerSET.CreationCHID = emptyCH.CHID;
db.CaseReviewerSets.Add(newCaseReviewerSET);
emptyCH.CaseReviewerSet = newCaseReviewerSET;
}
db.SaveChanges();
}
tran.Complete();
}
return response;
}
The exception occures on "db.SaveChanges()"
I saw in another post with the same error message something about "it seems I cannot have two connections opened to the same database with the TransactionScope block." but I dont think that this has anything to do with my case.
Additionally the number of records to insert and update in total are 2700, witch is not that many really. But it does take quite a lot of time to complete the for statement (10 minutes or so). Since everything happening within the for statement is actually happening in the memory can someone please explane why is this taking so long ?
You can try as shown below using latest db.Database.BeginTransaction API.
Note : use foreach instead of for
using (var db = new Entities.WaveEntities())
{
using (var dbContextTransaction = db.Database.BeginTransaction())
{
try
{
var maxCaseReviewersSetID = db.CaseReviewerSets.Select(crs => crs.CaseReviewersSetId).Max();
var emptyCHList = db.CaseHistories.Where(ch => ch.CaseReviewersSetID == null && ch.IsLatest == true && ch.StatusID != 100).ToList();
foreach(var ch in emptyCHList) {
var newCaseReviewerSET = new Entities.CaseReviewerSet();
newCaseReviewerSET.CreationCHID = ch.CHID;
db.CaseReviewerSets.Add(newCaseReviewerSET);
}
db.SaveChanges();
dbContextTransaction.Commit();
}
catch (Exception)
{
dbContextTransaction.Rollback();
}
}
}
I've just started a Windows Phone app, and I need to get all the user's followings.
I tried this :
SharedState.Authorizer = pinAuth;
ITwitterAuthorizer auth = SharedState.Authorizer;
TwitterContext twitterCtx = new TwitterContext(auth);
var friendList =
(from friend in twitterCtx.SocialGraph
where friend.Type == SocialGraphType.Friends && friend.ScreenName == "_TDK"
select friend)
.SingleOrDefault();
List<String> Followings;
foreach (var id in friendList.ScreenName)
{
Followings.Add(id.ToString());
}
But friendlist is always null and, obviously, the foreach does not like that and throws an exception.
Could someone help me ?
Thanks.
I think you need to iterate over the IDs collection, like this:
foreach (var id in friendList.IDs)
{
Followings.Add(id.ToString());
}
You need to make async calls with Silverlight-based apps, including Windows Phone. Here's an example of how you can refactor the query:
var twitterCtx = new TwitterContext(auth);
(from social in twitterCtx.SocialGraph
where social.Type == SocialGraphType.Followers &&
social.ScreenName == "JoeMayo"
select social)
.MaterializedAsyncCallback(asyncResponse =>
Dispatcher.BeginInvoke(() =>
{
if (asyncResponse.Status != TwitterErrorStatus.Success)
{
MessageBox.Show(
"Error during query: " +
asyncResponse.Exception.Message);
return;
}
SocialGraph social = asyncResponse.State.SingleOrDefault();
SocialListBox.ItemsSource = social.IDs;
}));
The MaterializedAsyncCallback manages the callback from Twitter. Notice how I use Dispatcher.BeginInvoke to marshal the call back onto the UI thread as the callback is on a worker thread. On the asyncResponse callback parameter, use Status to see if there is an error and use State to get the data if the query is successful.
I had the same problem, I solved this way (I know it's not the best way)
public void getProfile(MyProgressBar myprogressbar)
{
var auth = new SingleUserAuthorizer
{
Credentials = new InMemoryCredentials
{
ConsumerKey = GlobalVariables.ConsumerKey,
ConsumerSecret = GlobalVariables.ConsumerSecret,
AccessToken = GlobalVariables.AccessToken,
OAuthToken = GlobalVariables.AccessTokenSecret
}
};
using (var twitterCtx = new TwitterContext(auth, "https://api.twitter.com/1/", "https://search.twitter.com/"))
{
//Log
twitterCtx.Log = Console.Out;
var queryResponse = (from tweet in twitterCtx.Status
where tweet.Type == StatusType.User && tweet.ScreenName == GlobalVariables.ScreenName
select tweet);
queryResponse.AsyncCallback(tweets =>
Deployment.Current.Dispatcher.BeginInvoke(() =>
{
var publicTweets = (from tweet in tweets
select tweet).FirstOrDefault();
s.TwitterName = publicTweets.User.Name.ToString();
s.TwitterScreenName = "#" + GlobalVariables.ScreenName;
s.TwitterDescription = publicTweets.User.Description.ToString();
s.TwitterStatus = publicTweets.User.StatusesCount.ToString() + " Tweets / " + publicTweets.User.FriendsCount.ToString() + " Following / " + publicTweets.User.FollowersCount.ToString() + " Followers";
s.TwitterImage = publicTweets.User.ProfileImageUrl.ToString();
myprogressbar.ShowProgressBar = false;
})).SingleOrDefault();
}
}