I use code to get data from Atlassian Jira and put the data from the response into a container, how can I make this method asynchronous?
I have tried various methods with webrequest, but they all point to an error in the code, perhaps I do not understand how it works at all. You can suggest documentation or specify in the code what I can do with asynchrony for such a request.
public IEnumerable<JiraDataModel> GetData(string dateFrom, string dateTo)
{
int allTicketsCount = 0;
int devTicketsCount = 0;
int slaTicketsCount = 0;
List<JiraRequestUrl> urlArray = new List<JiraRequestUrl>();
urlArray.AddRange(new List<JiraRequestUrl>
{
new JiraRequestUrl{type = "all", //*data*//},
new JiraRequestUrl(){type = "dev",//*data*//});
try
{
foreach (JiraRequestUrl u in urlArray)
{
WebRequest request = WebRequest.Create(_jiraUrl + u.url);
request.ContentType = "application/json; charset=utf-8";
request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(Encoding.Default.GetBytes(_credentials)));
request.Headers.Add("maxResults", "100");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
JiraGetDataModel.RootObject DeserializedResponse = (JiraGetDataModel.RootObject)JsonConvert.DeserializeObject(responseString, typeof(JiraGetDataModel.RootObject));
if (u.type.Equals("all"))
{
allTicketsCount = DeserializedResponse.total;
}
if (u.type.Equals("dev"))
{
devTicketsCount = DeserializedResponse.total;
}
}
}
sorry, I didn't quite understand that I didn't insert my attempts to use async, I did it, and it seems to work, can I improve something in my code? added 2 lines
public async Task<IEnumerable<JiraDataModel>> GetData(string dateFrom, string dateTo)
{
int allTicketsCount = 0;
int devTicketsCount = 0;
int slaTicketsCount = 0;
List<JiraRequestUrl> urlArray = new List<JiraRequestUrl>();
urlArray.AddRange(new List<JiraRequestUrl>
{
new JiraRequestUrl{type = "all", //*data*//},
new JiraRequestUrl(){type = "dev",//*data*//});
try
{
foreach (JiraRequestUrl u in urlArray)
{
WebRequest request = WebRequest.Create(_jiraUrl + u.url);
request.ContentType = "application/json; charset=utf-8";
request.Headers.Add("Authorization", "Basic " + Convert.ToBase64String(Encoding.Default.GetBytes(_credentials)));
request.Headers.Add("maxResults", "100");
HttpWebResponse response = (HttpWebResponse)(await request.GetResponseAsync()request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
JiraGetDataModel.RootObject DeserializedResponse = (JiraGetDataModel.RootObject)JsonConvert.DeserializeObject(responseString, typeof(JiraGetDataModel.RootObject));
if (u.type.Equals("all"))
{
allTicketsCount = DeserializedResponse.total;
}
if (u.type.Equals("dev"))
{
devTicketsCount = DeserializedResponse.total;
}
}
}
I am struggling with the POST request for creating an article.
Can anybody please provide me an example value of the canonical request in C#?
The error I receive is signature issue:
{"errors":[{"code":"WRONG_SIGNATURE"}]}
From what I learnt a canonical request is build from:
Method = POST
URL = https://news-api.apple.com/channels/ChanelID/articles
Date = 2019-07-24T18:12:32Z
Content-Type = multipart/form-data
Body - Not fully understood
The Apple News API documentation says
If the request is a POST request and it includes an entity, include
the following:
The value of the Content-Type header
The full content of the entity - What is meant by this?
I want to post one test article from the json files provided in the apple's documentation.
This is the class generating the auth header:
public class Security
{
public static string AuthHeader(string method, string url, object content=null)
{
string formDataBoundary = String.Format("{0:N}", Guid.NewGuid());
var apiKeyId = Constants.AppleNewsKeyId;
var apiKeySecret = Constants.AppleNewsKeySecret;
if (string.IsNullOrEmpty(apiKeyId) || string.IsNullOrEmpty(apiKeySecret)) return string.Empty;
var encoding = new ASCIIEncoding();
var dt = DateTime.Now.ToString(Constants.DateFormat);
var canonicalRequest = string.Format("{0}{1}{2}{3}", method, url, dt, content);
var key = Convert.FromBase64String(apiKeySecret);
var hmac = new HMACSHA256(key);
var hashed = hmac.ComputeHash(encoding.GetBytes(canonicalRequest));
var signature = Convert.ToBase64String(hashed);
var authorizaton = string.Format(#"HHMAC; key={0}; signature={1}; date={2}", apiKeyId, signature, dt);
return authorizaton;
}
}
This is the constants class:
public static class Constants
{
public static readonly string ChannelId = "myID";
public static readonly string AppleNewsBaseUri = "https://news-api.apple.com";
public static readonly string DateFormat = "yyyy-MM-ddTHH:mm:ssK";
public static readonly string AppleNewsKeySecret = "myID";
public static readonly string AppleNewsKeyId = "myID";
}
And the action class which is missing code in order to have proper content instead of null passed to the AuthHeader method:
public class Action
{
public static string SendCommand(string action, string method)
{
var url = $"{Constants.AppleNewsBaseUri}{action}" + "/articles";
//string content = String.Format("{0}:{1}", "Content-Disposition", "form-data; filename=article.json; name=article.json;");
var authheader = Security.AuthHeader(method, url, null);
var request = WebRequest.Create(url);
request.PreAuthenticate = true;
request.Method = "POST";
request.Headers.Add("Authorization", authheader);
if (method.Equals("post", StringComparison.InvariantCultureIgnoreCase))
request.ContentType = "multipart/form-data;" ;
var output = string.Empty;
try
{
string filePath = Path.GetFullPath("article.json");
using (StreamReader r = new StreamReader(filePath))
{
string json = r.ReadToEnd();
dynamic jsonObj = JsonConvert.DeserializeObject(json);
ASCIIEncoding encoding = new ASCIIEncoding();
Byte[] bytes = encoding.GetBytes(json);
request.ContentLength = bytes.Length;
Stream newStream = request.GetRequestStream();
newStream.Write(bytes, 0, bytes.Length);
newStream.Close();
}
}
catch (Exception e)
{
Console.WriteLine("The file could not be read:");
Console.WriteLine(e.Message);
}
try
{
using (var response = request.GetResponse())
{
using (var reader = new StreamReader(response.GetResponseStream()))
output = reader.ReadToEnd();
}
}
catch (WebException e)
{
using (var reader = new StreamReader(e.Response.GetResponseStream()))
{
output = reader.ReadToEnd();
}
}
return output;
}
public static string ReadChannel()
{
var action = $"/channels/{Constants.ChannelId}";
const string method = "POST";
return SendCommand(action, method);
}
}
I need to download numerous web pages' sources. So I need to do that as fast as possible. Here is my codes.
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
WebResponse res = await siteyeBaglantiTalebi.GetResponseAsync();
StreamReader read = new StreamReader(res.GetResponseStream());
return read.ReadToEnd();
}
List<string> links = new List<string>(){... including some web page links};
private static List<string> source_list(List<string> links)
{
List<string> sources = new List<string>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloaded : " + i);
sources.Add(_task.Result);
}
return sources;
}
I was wondering if this code is the fastest way or it can be enhanced.
Can u pls help me with that ?
You are performing a _task.Result call inside each loop. Your code will run as fast as if you downloaded each page one after another if you code it like that.
Try this instead:
private async static Task<List<string>> source_list(List<string> links)
{
List<Task<string>> sources = new List<Task<string>>();
for (int i = 0; i < links.Count; i++)
{
Task<string> _task = downloadsource(links[i]);
Console.WriteLine("Downloading : " + i);
sources.Add(_task);
}
return (await Task.WhenAll(sources)).ToList();
}
This would be even better:
private async static Task<string[]> source_list(List<string> links)
{
return await Task.WhenAll(links.Select(l => downloadsource(l)));
}
Also I cleaned up your downloadsource method:
private static async Task<string> downloadsource(string link)
{
ServicePointManager.Expect100Continue = false;
WebRequest req = WebRequest.Create(link);
req.Proxy = null;
req.Method = "GET";
using (WebResponse res = await req.GetResponseAsync())
{
using (StreamReader read = new StreamReader(res.GetResponseStream()))
{
return read.ReadToEnd();
}
}
}
I am receiving multiple records when I expect one record which is the second to last record of json data.
Does anyone have any suggestions on what I might be doing incorrectly?
Sample data:
{
"timestamp":"2017-03-17T17:33:09.156Z",
"dataFrame":"VGVtcDoyOC43MA==",
"fcnt":318,
"port":2,
"rssi":-107,
"snr":0.8,
"sf_used":7,
"id":1489771989156,
"decrypted":true
},
{
"timestamp":"2017-03-17T17:33:16.614Z",
"dataFrame":"SHVtOjQzLjA0",
"fcnt":319,
"port":2,
"rssi":-108,
"snr":2.8,
"sf_used":7,
"id":1489771996614,
"decrypted":true
}
Code:
public ActionResult pir()
{
List<PIRDetailsViewModel> model = new List<PIRDetailsViewModel>();
model.AddRange(getPirDetails("0000acfffe588041"));
model = model.OrderByDescending(x => x.UpdatedTime).ToList();
return View(model);
}
private List<PIRDetailsViewModel> getPirDetails(string deviceID)
{
List<PIRDetailsViewModel> model = new List<PIRDetailsViewModel>();
try
{
WebRequest req = WebRequest.Create(#"https:example.com");
req.Method = "GET";
req.Headers["Authorization"] = "Basic " + "password==";
HttpWebResponse resp = req.GetResponse() as HttpWebResponse;
var encoding = resp.CharacterSet == "" ? Encoding.UTF8 : Encoding.GetEncoding(resp.CharacterSet);
using (var stream = resp.GetResponseStream())
{
var reader = new StreamReader(stream, encoding);
var responseString = reader.ReadToEnd();
List<PIRDetail> Pirs = Newtonsoft.Json.JsonConvert.DeserializeObject <List<PIRDetail>>(responseString);
foreach (var item in Pirs)
{
byte[] data = Convert.FromBase64String(item.dataFrame.ToString());
PIRDetailsViewModel binModel = new PIRDetailsViewModel();
binModel.deviceid = deviceID;
binModel.status = Convert.ToString(Encoding.UTF8.GetString(data).Substring(4));
binModel.UpdatedTime = TimeZoneInfo.ConvertTimeFromUtc(item.timestamp, TimeZoneInfo.FindSystemTimeZoneById("India Standard Time")).ToString();
model.Add(binModel);
}
}
}
catch (Exception Ex)
{
}
return model;
}
The above code only returns multiple records, but I am trying to get the second to last record that is being returned each time this code is ran.
I'm rewriting. Based on comments, I think you want something more like below (loop not necessary):
List<PIRDetail> Pirs = Newtonsoft.Json.JsonConvert.DeserializeObject <List<PIRDetail>>(responseString);
var item = Pirs[Pirs.Count -2];
byte[] data = Convert.FromBase64String(item.dataFrame.ToString());
PIRDetailsViewModel binModel = new PIRDetailsViewModel();
binModel.deviceid = deviceID;
binModel.status = Convert.ToString(Encoding.UTF8.GetString(data).Substring(4));
binModel.UpdatedTime = TimeZoneInfo.ConvertTimeFromUtc(item.timestamp, TimeZoneInfo.FindSystemTimeZoneById("India Standard Time")).ToString();
model.Add(binModel);
I'm connecting to a web service from .NET, like:
var request = (HttpWebRequest) WebRequest.Create(uri);
request.Credentials = new NetworkCredential("usr", "pwd", "domain");
var response = (HttpWebResponse) request.GetResponse();
The authorization header looks like:
Authorization: Digest username="usr",realm="domain",nonce="...",
uri="/dir",algorithm="MD5",etc...
^^^^^^^^^^
The server returns (400) Bad Request. A header send by Chrome or IE looks like:
Authorization: Digest username="usr", realm="domain", nonce="...",
uri="/dir/query?id=1", algorithm=MD5, etc...
^^^^^^^^^^^^^^^^^^^^^
We suspect that the difference in URI is causing the web service to deny the request with a 400 error. Is it possible to make HttpRequest send out an Authorization header that includes the full URI?
It turns out that Digest authentication is fairly easy to implement. With our own implementation, we were able to use the full URI (including parameters) to generate the MD5 hash. That fixed the problem.
In case someone hits this problem in the future, you can call the workaround like:
var resultText = DigestAuthFixer.GrabResponse("/dir/index.html");
The code for the DigestAuthFixer class:
public static class DigestAuthFixer
{
private static string _host = "http://localhost";
private static string _user = "Mufasa";
private static string _password = "Circle Of Life";
private static string _realm;
private static string _nonce;
private static string _qop;
private static string _cnonce;
private static DateTime _cnonceDate;
private static int _nc;
private static string CalculateMd5Hash(
string input)
{
var inputBytes = Encoding.ASCII.GetBytes(input);
var hash = MD5.Create().ComputeHash(inputBytes);
var sb = new StringBuilder();
foreach (var b in hash)
sb.Append(b.ToString("x2"));
return sb.ToString();
}
private static string GrabHeaderVar(
string varName,
string header)
{
var regHeader = new Regex(string.Format(#"{0}=""([^""]*)""", varName));
var matchHeader = regHeader.Match(header);
if (matchHeader.Success)
return matchHeader.Groups[1].Value;
throw new ApplicationException(string.Format("Header {0} not found", varName));
}
// http://en.wikipedia.org/wiki/Digest_access_authentication
private static string GetDigestHeader(
string dir)
{
_nc = _nc + 1;
var ha1 = CalculateMd5Hash(string.Format("{0}:{1}:{2}", _user, _realm, _password));
var ha2 = CalculateMd5Hash(string.Format("{0}:{1}", "GET", dir));
var digestResponse =
CalculateMd5Hash(string.Format("{0}:{1}:{2:00000000}:{3}:{4}:{5}", ha1, _nonce, _nc, _cnonce, _qop, ha2));
return string.Format("Digest username=\"{0}\", realm=\"{1}\", nonce=\"{2}\", uri=\"{3}\", " +
"algorithm=MD5, response=\"{4}\", qop={5}, nc={6:00000000}, cnonce=\"{7}\"",
_user, _realm, _nonce, dir, digestResponse, _qop, _nc, _cnonce);
}
public static string GrabResponse(
string dir)
{
var url = _host + dir;
var uri = new Uri(url);
var request = (HttpWebRequest)WebRequest.Create(uri);
// If we've got a recent Auth header, re-use it!
if (!string.IsNullOrEmpty(_cnonce) &&
DateTime.Now.Subtract(_cnonceDate).TotalHours < 1.0)
{
request.Headers.Add("Authorization", GetDigestHeader(dir));
}
HttpWebResponse response;
try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException ex)
{
// Try to fix a 401 exception by adding a Authorization header
if (ex.Response == null || ((HttpWebResponse)ex.Response).StatusCode != HttpStatusCode.Unauthorized)
throw;
var wwwAuthenticateHeader = ex.Response.Headers["WWW-Authenticate"];
_realm = GrabHeaderVar("realm", wwwAuthenticateHeader);
_nonce = GrabHeaderVar("nonce", wwwAuthenticateHeader);
_qop = GrabHeaderVar("qop", wwwAuthenticateHeader);
_nc = 0;
_cnonce = new Random().Next(123400, 9999999).ToString();
_cnonceDate = DateTime.Now;
var request2 = (HttpWebRequest)WebRequest.Create(uri);
request2.Headers.Add("Authorization", GetDigestHeader(dir));
response = (HttpWebResponse)request2.GetResponse();
}
var reader = new StreamReader(response.GetResponseStream());
return reader.ReadToEnd();
}
}
I ran into this issue recently. I couldn't get the workaround from Andomar to work either without some minor adjustments. I submitted the changes as a suggestion to Andomar's answer, but they were unceremoniously rejected by TheTinMan and Lucifer. Since it took me and some collegues hours to figure these out and I'm sure someone else will need this I'm posting the code as an answer to make it available.
Here's the adjusted code. Basically an "opaque" header variable was needed, and some quotes needed fixing in GetDigestHeader.
public static class DigestAuthFixer
{
private static string _host = "http://localhost";
private static string _user = "Mufasa";
private static string _password = "Circle Of Life";
private static string _realm;
private static string _nonce;
private static string _qop;
private static string _cnonce;
private static string _opaque;
private static DateTime _cnonceDate;
private static int _nc = 0;
private static string CalculateMd5Hash(
string input)
{
var inputBytes = Encoding.ASCII.GetBytes(input);
var hash = MD5.Create().ComputeHash(inputBytes);
var sb = new StringBuilder();
foreach (var b in hash)
sb.Append(b.ToString("x2"));
return sb.ToString();
}
private static string GrabHeaderVar(
string varName,
string header)
{
var regHeader = new Regex(string.Format(#"{0}=""([^""]*)""", varName));
var matchHeader = regHeader.Match(header);
if (matchHeader.Success)
return matchHeader.Groups[1].Value;
throw new ApplicationException(string.Format("Header {0} not found", varName));
}
// http://en.wikipedia.org/wiki/Digest_access_authentication
private static string GetDigestHeader(
string dir)
{
_nc = _nc + 1;
var ha1 = CalculateMd5Hash(string.Format("{0}:{1}:{2}", _user, _realm, _password));
var ha2 = CalculateMd5Hash(string.Format("{0}:{1}", "GET", dir));
var digestResponse =
CalculateMd5Hash(string.Format("{0}:{1}:{2:00000000}:{3}:{4}:{5}", ha1, _nonce, _nc, _cnonce, _qop, ha2));
return string.Format("Digest username=\"{0}\", realm=\"{1}\", nonce=\"{2}\", uri=\"{3}\", " +
"algorithm=MD5, response=\"{4}\", qop=\"{5}\", nc=\"{6:00000000}\", cnonce=\"{7}\", opaque=\"{8}\"",
_user, _realm, _nonce, dir, digestResponse, _qop, _nc, _cnonce, _opaque);
}
public static string GrabResponse(
string dir)
{
var url = _host + dir;
var uri = new Uri(url);
var request = (HttpWebRequest)WebRequest.Create(uri);
// If we've got a recent Auth header, re-use it!
if (!string.IsNullOrEmpty(_cnonce) &&
DateTime.Now.Subtract(_cnonceDate).TotalHours < 1.0)
{
request.Headers.Add("Authorization", GetDigestHeader(dir));
}
HttpWebResponse response;
try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException ex)
{
// Try to fix a 401 exception by adding a Authorization header
if (ex.Response == null || ((HttpWebResponse)ex.Response).StatusCode != HttpStatusCode.Unauthorized)
throw;
var wwwAuthenticateHeader = ex.Response.Headers["WWW-Authenticate"];
_realm = GrabHeaderVar("realm", wwwAuthenticateHeader);
_nonce = GrabHeaderVar("nonce", wwwAuthenticateHeader);
_qop = GrabHeaderVar("qop", wwwAuthenticateHeader);
_opaque = GrabHeaderVar("opaque", wwwAuthenticateHeader);
_nc = 0;
_cnonce = new Random().Next(123400, 9999999).ToString();
_cnonceDate = DateTime.Now;
var request2 = (HttpWebRequest)WebRequest.Create(uri);
request2.Headers.Add("Authorization", GetDigestHeader(dir));
response = (HttpWebResponse)request2.GetResponse();
}
var reader = new StreamReader(response.GetResponseStream());
return reader.ReadToEnd();
}
}
It looks like you need to install this hotfix might help you out:
http://support.microsoft.com/?kbid=924638
Your problem was probably happening because you weren't able to set the KeepAlive property to false when you're using the HTTP adapter to post a message
Also ensure PreAuthenticate is set to true.