So I am quickly learning the ways of C# (complete noob that inherited this problem); I have written up the following code which calls a web service that returns JSON that is not always well-formed. The mission here is to take the JSON string and break it into array segments which get inserted into an SQL table for further parsing and testing. I.e. if the return string was something like
{1234:{5678:{1:{"key":"val","key":"val"},{2:{"key":"val","key":"val"}}}}
then the rows would be:
{1234}
{5678}
{1:{"key":"val","key":"val"}
{2:{"key":"val","key":"val"}
This is .NET 3.0 and SQL Server 2008 R2 (Legacy stuff).
Here is my working code:
public partial class UserDefinedFunctions
{
[Microsoft.SqlServer.Server.SqlFunction(DataAccess =
DataAccessKind.Read)]
public static SqlString TestParse(SqlString uri, SqlString username, SqlString passwd, SqlString postdata)
{
//-----
// The SqlPipe is how we send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
SqlString document;
try
{
// Set up the request, including authentication
WebRequest req = WebRequest.Create(Convert.ToString(uri));
if (Convert.ToString(username) != null & Convert.ToString(username) != "")
{
req.Credentials = new NetworkCredential(
Convert.ToString(username),
Convert.ToString(passwd));
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
//SqlContext.Pipe.Send("...get the data");
using (StreamReader rdr = new StreamReader(dataStream))
{
document = (SqlString)rdr.ReadToEnd();
rdr.Close();
//-----
string connectionString = null;
string sql = null;
connectionString = "Data source= 192.168.0.5; Database=Administration;User Id=Foo;Password=Blah; Trusted_Connection=True;";
using (SqlConnection cnn = new SqlConnection(connectionString))
{
sql = "INSERT INTO JSON_DATA (JSONROW) VALUES(#data)";
cnn.Open();
using (SqlCommand cmd = new SqlCommand(sql, cnn))
{
String payload = "";
String nestpayload = "";
int nests = 0;
String json = document.ToString();
/*first lets do some housekeeping on our payload; double closing curly braces need to be escaped (with curly braces!) in order to keep them in the string.*/
json = json.Replace("\\", "");
int i = json.Length;
//return new SqlString(json);
while (i > 1)
{
/*find the first closing "}" in the string and then check to see if there are more than one.
We need to read the data up to each closing brace, pull off that substring and process it for each iteration until the string is gone.*/
int closingbrace = json.IndexOf("}"); //First closing brace
int nextbrace = Math.Max(0, json.IndexOf("{", closingbrace)); //Next opening brace
String ChkVal = json.Substring(closingbrace + 1, Math.Max(1, nextbrace - closingbrace)); //+1 to ignore the 1st closing brace
int checks = Math.Max(0, ChkVal.Length) - Math.Max(0, ChkVal.Replace("}", "").Length);
payload = json.Substring(0, Math.Max(0, (json.IndexOf("}") + 1)));
/*Remove the payload from the string*/
json = json.Substring(payload.Length + 1);
/*"nests" is how many nested levels excluding the opening brace for the closing brace we found.*/
nests = (payload.Length - payload.Replace("{", "").Length);
/*If we have more then one nest level check to see if any of them go with the payload*/
if (nests > 1)
{
/*Break out the nested section and remove it from the payload.*/
nestpayload = payload.Substring(0, payload.LastIndexOf("{"));
payload = payload.Substring(payload.LastIndexOf("{"), payload.Length - payload.LastIndexOf("{"));
while (nests > 1)
{
if (checks > 0) //# of right braces in payload equals number of left-side nests go with the payload
{
// payload = nestpayload.Substring(Math.Max(0, nestpayload.LastIndexOf("{")), Math.Max(0, nestpayload.Length) - Math.Max(0, (nestpayload.LastIndexOf("{")))) + payload;//The second Math.Max defaults to 1; if we got here there is at minimum one "{" character in the substring
payload = nestpayload.Substring(nestpayload.LastIndexOf("{")) + payload;
nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0, nestpayload.LastIndexOf("{"))));
checks--;
nests--;
}
else
{
/*If we got here there are no more pieces of the nested data to append to the payload.
We use an array and string.split to keep the nest ordering correct.*/
string[] OrderedNest = nestpayload.Split('{');
for (int s = 0; s < OrderedNest.Length; s++)
{
if (OrderedNest[s] != "")
{
cmd.Parameters.AddWithValue("#data", "{" + OrderedNest[s].Replace(":", "}"));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
}
//cmd.Parameters.AddWithValue("#data", nestpayload.Substring(Math.Max(0,nestpayload.LastIndexOf("{"))).Replace(":","}"));
//cmd.Parameters.AddWithValue("#data", OrderedNest[1].Replace(":","}")+OrderedNest[2]);
// cmd.ExecuteNonQuery();
//cmd.Parameters.Clear();
//nests = Math.Max(0, nests - 1);
nests = 0;
//nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0,nestpayload.LastIndexOf("{"))));
}
}
}
/*At the very end payload will be a single "}"; check for this and discard the last row*/
if (payload != "}")
{
cmd.Parameters.AddWithValue("#data", new SqlChars(payload));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
/*Get the new string length*/
i = json.Length;
payload = "";
}
}
}
//-----
/* }
catch (Exception e)
{
return e.ToString();
}*/
}
// Close up everything...
dataStream.Close();
}
resp.Close();
// .. and return the output to the caller.
}//end using
return ("Finished");
}
catch (WebException e)
{
throw e;
}
}
}
While it works, it is INCREDIBLY slow; 4+ minutes to write 1500 rows to the server. Once daily this will need to write ~60,000 records in; the rest of the time it will be maybe 100 records POSTED and returned (I haven't worked up the POST part yet). I'm sure there are plenty of things I am doing not-so-proper here that are causing issues, but I have absolutely no idea where to even begin. I was excited enough that I could get the right response out of this! Any ideas/thoughts/help/sympathy would be greatly appreciated.
There are several problems here, not the least of which is that it appears you have posted your "sa" password to these here public interwebs. Here are the code issues that I see:
While it is possible to do Web Services calls in SQLCLR, it is definitely an advanced topic, full of pitfalls. This is not something that should be undertaken by a novice / beginner in SQLCLR, which itself is already a nuanced subset of regular .NET programming.
Get rid of the SqlPipe line and the comment line above it. Functions do not pass data back to the caller via SqlPipe; that is for Stored Procedures.
You probably shouldn't be using WebRequest
document should be string, not SqlString. You never return document and only ever convert it back to string, so it should just be that.
Use HttpWebRequest instead of WebRequest. This way you won't have to occasionally cast it into HttpWebRequest.
Don't convert the SqlString input parameters into string (e.g. Convert.ToString(uri)). All Sql* types have a Value property that returns the value in the native .NET type. So instead just use uri.Value, and so on.
Don't check for NULL inputs via Convert.ToString(username) != null. All Sql* type have an IsNull property that you can check. So instead use !username.IsNull.
DO NOT do all of your text processing (especially processing that contacts another system to do row-by-row inserts) while keeping the remote HttpWebRequest connection open. The only thing you should be doing within the using (WebResponse resp = req.GetResponse()) is populating the document variable. Don't do any processing of the contents of document until you are outside of that outer-most using().
Don't do individual inserts (i.e. the while (i > 1) loop). They aren't even in a transaction. If you get an error in the middle of the document, you will have loaded partial data (unless that's ok for this process).
ALWAYS Schema-qualify database objects. Meaning, JSON_DATA should be dbo.JSON_DATA (or whatever Schema is being used if not dbo).
In your connectionString you have both Id/Password and Trusted_Connection. Don't use both as they are mutually exclusive options (if you have both, the Id/Password are ignored and only Trusted_Connection is used).
Please, please do not log in as sa or have your application log in as sa. That is just begging for a disaster.
Are you connecting to a different instance of SQL Server than this SQLCLR object is running on? If it is the same instance, you might be better off changing this into a SqlProcedure so that you can use Context_Connection=True; as the connection string. That is the in-process connection that attaches to the session that it is being called from.
Don't use Parameters.AddWithValue(). Bad idea. Create the SqlParameter with the specific, and appropriate, datatype. Then add is to the Parameters collection via Add().
There might be other issues, but these were the obvious ones. As I said in point #1, you might be in over your head here. Not trying to be negative, just trying to avoid another poor implementation of SQLCLR that often leads to negative views of this otherwise very useful feature. If you want to pursue this, then please do more research first into how SQLCLR works, best practices, etc. A good place to start is a series that I am writing on this topic on SQL Server Central: Stairway to SQLCLR.
Or, another option is to use the INET_GetWebPages SQLCLR TVF that is available in the Full version of the SQL# SQLCLR library (which I wrote). This option is not free, but it would allow you to simply install the web request piece and then you just need to parse the returned document separately in a SQLCLR scalar UDF (which is probably the best approach anyway, even if you do the web request function / stored procedure on your own). In fact, if you are inserting into a table in the same Instance of SQL Server, you can create a SQLCLR TVF for the document parser and pass each OrderedNest value back using yield return (to stream the results back) and use as follows:
DECLARE #JSON NVARCHAR(MAX);
SELECT #JSON = [content]
FROM SQL#.INET_GetWebPages(#uri, .....);
INSERT INTO dbo.JSON_DATA (JSONROW)
SELECT [column_name]
FROM dbo.MyBrokenJsonFixerUpper(#JSON);
Good luck!
I am marking this question answered as it has become clear that what was need is a rewrite and rethinking of my original script. #Solomon Rutzky upvoted for providing helpful information which pointed me to this conclusion. For those
interested here is the rewrite:
using System;
using System.Data;
using System.Data.SqlClient;
using System.Data.SqlTypes;
using Microsoft.SqlServer.Server;
using System.Collections;
using System.Globalization;
// Other things we need for WebRequest
using System.Net;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
public partial class StoredProcedures
{
[Microsoft.SqlServer.Server.SqlProcedure]
public static void ApiParser(SqlString uri, SqlString user, SqlString pwd, SqlString postd)
{
// Create an SqlPipe to send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
//Make sure we have a url to process
if (uri.IsNull || uri.Value.Trim() == string.Empty)
{
pipe.Send("uri cannot be empty");
return;
}
try
{
//Create our datatable and get the table structure from the database
DataTable table = new DataTable();
string connectionString = null;
//connectionString = "Data source= 192.168.0.5; Database=Administration; Trusted_Connection=True;";
connectionString = "Data Source=(localdb)\\ProjectsV12;Integrated Security=True;Connect Timeout=30;Encrypt=False;TrustServerCertificate=False;ApplicationIntent=ReadWrite;MultiSubnetFailover=False";
using (SqlConnection gts = new SqlConnection(connectionString))
{
gts.Open();
using (SqlDataAdapter adapter = new SqlDataAdapter("SELECT TOP 0 * FROM sp_WebSvcs.dbo.JSON_DATA", gts))
{
adapter.Fill(table);
}
}
// Send a message string back to the client.
pipe.Send("Beginning Api Call...");
String json = "";
// Set up the request, including authentication
WebRequest req = HttpWebRequest.Create(uri.Value);
if (!user.IsNull & user.Value != "")
{
req.Credentials = new NetworkCredential(user.Value, pwd.Value);
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
using (StreamReader rdr = new StreamReader(dataStream))
{
json = (String)rdr.ReadToEnd();
rdr.Close();
}
// Close up everything...
dataStream.Close();
}
resp.Close();
}//end using resp
pipe.Send("Api Call complete; Parsing returned data...");
int i = 0;
String h = "";
String l = "";
int s = 0;
int p = 0;
int b = 0;
int payload = 0;
foreach (string line in json.Split(new[] { "}," }, StringSplitOptions.None))
{
if (line != "")
{
l = line;
i = l.Replace("{", "").Length + 1;
p = l.LastIndexOf("{");
if (line.Length > i) //we find this at the beginning of a group of arrays
{
h = line.Substring(0, p - 1);
s = Math.Max(0, h.LastIndexOf("{"));
if (h.Length > s && s != 0)
/*We have a nested array that has more than one level.
*This should only occur at the beginning of new array group.
*Advance the payload counter and get the correct string from line.*/
{
payload++;
l = line.Substring(s, line.Length - s);
}
h = (s >= 0) ? h.Substring(0, s) : h;
//=============
/*At this point 'h' is a nest collection. Split and add to table.*/
string[] OrderedNest = h.Split('{');
for (int z = 0; z < OrderedNest.Length; z++)
{
if (OrderedNest[z] != "")
{
table.Rows.Add(payload, "{" + OrderedNest[z].Replace(":", "").Replace("[","").Replace("]","") + "}");
}
}
//=============
}
else
{
h = null;
}
//at this point the first character in the row should be a "{"; If not we need to add one.
if (l[0].ToString() != "{")
{
l = "{" + l;
}
if (l.Replace("{", "").Length != l.Replace("}", "").Length) //opening and closing braces don't match; match the closing to the opening
{
l = l.Replace("}", "");
b = l.Length - l.Replace("{", "").Length;
l = l + new String('}', b);
}
table.Rows.Add(payload, l.Replace("\\\"", "").Replace("\\", "").Replace("]","").Replace("[",""));
}
}
//====
using (SqlConnection cnn = new SqlConnection(connectionString))
{
cnn.Open();
using (SqlBulkCopy copy = new SqlBulkCopy(cnn))
{
copy.DestinationTableName = "sp_WebSvcs.dbo.JSON_DATA";
copy.WriteToServer(table);
}
}
//====
} //end try
catch (Exception e)
{
pipe.Send("We have a problem!");
throw new Exception("\n\n" + e.Message + "\n\n");
}
pipe.Send("Parsing complete");
}
}
Related
I am trying to check the value of the max_allowed_packet size. This is my code:
public int MaxAllowedPacket()
{
var max = 0;
using (var conn = new MySqlConnection(_ConnectionString))
{
var sql = conn.CreateCommand();
sql.CommandText = "SHOW VARIABLES like 'max_allowed_packet'";
try
{
conn.Open();
var reader = sql.ExecuteReader();
// not sure where to go from here
}
catch (Exception ex)
{
// I've got some logging here
}
}
return max;
}
I'm suspecting the format of the query or the execution is wrong because my result is always
-1
EDIT:
I have edited the code to use sql.ExecuteReader() but the result is now:
"Enumeration yielded no results".
Eventually figured it out myself, and thought to post it here, before this gets downvoted even more...
var reader = sql.ExecuteReader();
reader.Read();
max = reader.GetInt32(1);
It's best to put some try catches around and you can optionally query the first field through reader.GetString(0), which should return "max_allowed_packet".
The following code gives me the error (I get it from the MessageBox.Show() in the catch block)
"Exception in PopulateBla() : There is a file sharing violation. A
different process might be using the file [,,,,,,]
CODE
using (SqlCeCommand cmd = new SqlCeCommand(SQL_GET_VENDOR_ITEMS, new SqlCeConnection(SQLCE_CONN_STR)))
{
cmd.Parameters.Add("#VendorID", SqlDbType.NVarChar, 10).Value = vendorId;
cmd.Parameters.Add("#VendorItemID", SqlDbType.NVarChar, 19).Value = vendorItemId;
try
{
cmd.Connection.Open();
using (SqlCeDataReader SQLCEReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
{
if (SQLCEReader.Read())
{
itemID = SQLCEReader.GetString(ITEMID_INDEX);
packSize = SQLCEReader.GetString(PACKSIZE_INDEX);
recordFound = true;
}
}
}
catch (SqlCeException err)
{
MessageBox.Show(string.Format("Exception in PopulateControlsIfVendorItemsFound: {0}\r\n", err.Message));//TODO: Remove
}
finally
{
if (cmd.Connection.State == ConnectionState.Open)
{
cmd.Connection.Close();
}
}
}
SQL_GET_VENDOR_ITEMS is my query string.
What file sharing problem could be happening here?
UPDATE
This is the kind of code that makes that sort of refactoring recommended by ctacke below difficult:
public void setINVQueryItemGroup( string ID )
{
try
{
dynSQL += " INNER JOIN td_item_group ON t_inv.id = td_item_group.id AND t_inv.pack_size = td_item_group.pack_size WHERE td_item_group.item_group_id = '" + ID + "'";
}
catch( Exception ex )
{
CCR.ExceptionHandler( ex, "InvFile.setINVQueryDept" );
}
}
A SQL statement is being appended to by means of a separate method, altering a global var (dynSQL) while possibly allowing for SQL Injection (depending on where/how ID is assigned). If that's not enough, any exception thrown could mislead the weary bughunter due to indicating it occurred in a different method (doubtless the victim of a careless copy-and-paste operation).
This is "Coding Horror"-worthy. How many best practices can you ignore in a scant few lines of code?
Here's another example:
string dynSQL = "SELECT * FROM purgatory WHERE vendor_item = '" + VendorItem + "' ";
if (vendor_id != "")
{
dynSQL += "AND vendor_id = '" + vendor_id + "' ";
}
It could be done by replacing the args with "?"s, but the code to then determine which/how many params to assign would be 42X uglier than Joe Garagiola's mean cleats.
I really like Chris' idea of using a single connection to your database. You could declare that global to your class like so:
public ClayShannonDatabaseClass
{
private SqlCeConnection m_openConnection;
public ClayShannonDatabaseClass()
{
m_openConnection = new SqlCeConnection();
m_openConnection.Open();
}
public void Dispose()
{
m_openConnection.Close();
m_openConnection.Dispose();
m_openConnection = null;
}
}
I'm guessing your code is crashing whenever you attempt to actually open the database.
To verify this, you could stick an integer value in the code to help you debug.
Example:
int debugStep = 0;
try
{
//cmd.Connection.Open(); (don't call this if you use m_openConnection)
debugStep = 1;
using (SqlCeDataReader SQLCEReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
{
debugStep = 2;
if (SQLCEReader.Read())
{
debugStep = 3;
itemID = SQLCEReader.GetString(ITEMID_INDEX);
debugStep = 4;
packSize = SQLCEReader.GetString(PACKSIZE_INDEX);
debugStep = 5;
recordFound = true;
}
}
}
catch (SqlCeException err)
{
string msg = string.Format("Exception in PopulateControlsIfVendorItemsFound: {0}\r\n", err.Message);
string ttl = string.Format("Debug Step: {0}", debugStep);
MessageBox.Show(msg, ttl); //TODO: Remove
}
// finally (don't call this if you use m_openConnection)
// {
// if (cmd.Connection.State == ConnectionState.Open)
// {
// cmd.Connection.Close();
// }
// }
I'm guessing your error is at Step 1.
Provided the file isn't marked read-only (you checked that, right?), then you have another process with a non-sharing lock on the file.
The isql.exe database browser that comes with SQL CE is a common culprit if it's running in the background.
Depending on your version of SQLCE, it's quite possible that another process has an open connection (can't recall what version started allowing multiple process connections), so if you have any other app in the background that has it open, that may be a problem too.
You're also using a boatload of connections to that database, and they don't always get cleaned up and released immediately up Dispose. I'd highly recommend building a simple connection manager class that keeps a single (or more like two) connections to the database and just reuses them for all operations.
I want to make the following curl call in my C# console application:
curl -d "text=This is a block of text" \
http://api.repustate.com/v2/demokey/score.json
I tried to do like the question posted here, but I cannot fill the properties properly.
I also tried to convert it to a regular HTTP request:
http://api.repustate.com/v2/demokey/score.json?text="This%20is%20a%20block%20of%20text"
Can I convert a cURL call to an HTTP request? If so, how? If not, how can I make the above cURL call from my C# console application properly?
Well, you wouldn't call cURL directly, rather, you'd use one of the following options:
HttpWebRequest/HttpWebResponse
WebClient
HttpClient (available from .NET 4.5 on)
I'd highly recommend using the HttpClient class, as it's engineered to be much better (from a usability standpoint) than the former two.
In your case, you would do this:
using System.Net.Http;
var client = new HttpClient();
// Create the HttpContent for the form to be posted.
var requestContent = new FormUrlEncodedContent(new [] {
new KeyValuePair<string, string>("text", "This is a block of text"),
});
// Get the response.
HttpResponseMessage response = await client.PostAsync(
"http://api.repustate.com/v2/demokey/score.json",
requestContent);
// Get the response content.
HttpContent responseContent = response.Content;
// Get the stream of the content.
using (var reader = new StreamReader(await responseContent.ReadAsStreamAsync()))
{
// Write the output.
Console.WriteLine(await reader.ReadToEndAsync());
}
Also note that the HttpClient class has much better support for handling different response types, and better support for asynchronous operations (and the cancellation of them) over the previously mentioned options.
Or in restSharp:
var client = new RestClient("https://example.com/?urlparam=true");
var request = new RestRequest(Method.POST);
request.AddHeader("content-type", "application/x-www-form-urlencoded");
request.AddHeader("cache-control", "no-cache");
request.AddHeader("header1", "headerval");
request.AddParameter("application/x-www-form-urlencoded", "bodykey=bodyval", ParameterType.RequestBody);
IRestResponse response = client.Execute(request);
Below is a working example code.
Please note you need to add a reference to Newtonsoft.Json.Linq
string url = "https://yourAPIurl";
WebRequest myReq = WebRequest.Create(url);
string credentials = "xxxxxxxxxxxxxxxxxxxxxxxx:yyyyyyyyyyyyyyyyyyyyyyyyyyyyyy";
CredentialCache mycache = new CredentialCache();
myReq.Headers["Authorization"] = "Basic " + Convert.ToBase64String(Encoding.ASCII.GetBytes(credentials));
WebResponse wr = myReq.GetResponse();
Stream receiveStream = wr.GetResponseStream();
StreamReader reader = new StreamReader(receiveStream, Encoding.UTF8);
string content = reader.ReadToEnd();
Console.WriteLine(content);
var json = "[" + content + "]"; // change this to array
var objects = JArray.Parse(json); // parse as array
foreach (JObject o in objects.Children<JObject>())
{
foreach (JProperty p in o.Properties())
{
string name = p.Name;
string value = p.Value.ToString();
Console.Write(name + ": " + value);
}
}
Console.ReadLine();
Reference: TheDeveloperBlog.com
Late response but this is what I ended up doing. If you want to run your curl commands very similarly as you run them on linux and you have windows 10 or latter do this:
public static string ExecuteCurl(string curlCommand, int timeoutInSeconds=60)
{
if (string.IsNullOrEmpty(curlCommand))
return "";
curlCommand = curlCommand.Trim();
// remove the curl keworkd
if (curlCommand.StartsWith("curl"))
{
curlCommand = curlCommand.Substring("curl".Length).Trim();
}
// this code only works on windows 10 or higher
{
curlCommand = curlCommand.Replace("--compressed", "");
// windows 10 should contain this file
var fullPath = System.IO.Path.Combine(Environment.SystemDirectory, "curl.exe");
if (System.IO.File.Exists(fullPath) == false)
{
if (Debugger.IsAttached) { Debugger.Break(); }
throw new Exception("Windows 10 or higher is required to run this application");
}
// on windows ' are not supported. For example: curl 'http://ublux.com' does not work and it needs to be replaced to curl "http://ublux.com"
List<string> parameters = new List<string>();
// separate parameters to escape quotes
try
{
Queue<char> q = new Queue<char>();
foreach (var c in curlCommand.ToCharArray())
{
q.Enqueue(c);
}
StringBuilder currentParameter = new StringBuilder();
void insertParameter()
{
var temp = currentParameter.ToString().Trim();
if (string.IsNullOrEmpty(temp) == false)
{
parameters.Add(temp);
}
currentParameter.Clear();
}
while (true)
{
if (q.Count == 0)
{
insertParameter();
break;
}
char x = q.Dequeue();
if (x == '\'')
{
insertParameter();
// add until we find last '
while (true)
{
x = q.Dequeue();
// if next 2 characetrs are \'
if (x == '\\' && q.Count > 0 && q.Peek() == '\'')
{
currentParameter.Append('\'');
q.Dequeue();
continue;
}
if (x == '\'')
{
insertParameter();
break;
}
currentParameter.Append(x);
}
}
else if (x == '"')
{
insertParameter();
// add until we find last "
while (true)
{
x = q.Dequeue();
// if next 2 characetrs are \"
if (x == '\\' && q.Count > 0 && q.Peek() == '"')
{
currentParameter.Append('"');
q.Dequeue();
continue;
}
if (x == '"')
{
insertParameter();
break;
}
currentParameter.Append(x);
}
}
else
{
currentParameter.Append(x);
}
}
}
catch
{
if (Debugger.IsAttached) { Debugger.Break(); }
throw new Exception("Invalid curl command");
}
StringBuilder finalCommand = new StringBuilder();
foreach (var p in parameters)
{
if (p.StartsWith("-"))
{
finalCommand.Append(p);
finalCommand.Append(" ");
continue;
}
var temp = p;
if (temp.Contains("\""))
{
temp = temp.Replace("\"", "\\\"");
}
if (temp.Contains("'"))
{
temp = temp.Replace("'", "\\'");
}
finalCommand.Append($"\"{temp}\"");
finalCommand.Append(" ");
}
using (var proc = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "curl.exe",
Arguments = finalCommand.ToString(),
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
WorkingDirectory = Environment.SystemDirectory
}
})
{
proc.Start();
proc.WaitForExit(timeoutInSeconds*1000);
return proc.StandardOutput.ReadToEnd();
}
}
}
The reason why the code is a little bit long is because windows will give you an error if you execute a single quote. In other words, the command curl 'https://google.com' will work on linux and it will not work on windows. Thanks to that method I created you can use single quotes and run your curl commands exactly as you run them on linux. This code also checks for escaping characters such as \' and \".
For example use this code as
var output = ExecuteCurl(#"curl 'https://google.com' -H 'Accept: application/json, text/javascript, */*; q=0.01'");
If you where to run that same string agains C:\Windows\System32\curl.exe it will not work because for some reason windows does not like single quotes.
I know this is a very old question but I post this solution in case it helps somebody. I recently met this problem and google led me here. The answer here helps me to understand the problem but there are still issues due to my parameter combination. What eventually solves my problem is curl to C# converter. It is a very powerful tool and supports most of the parameters for Curl. The code it generates is almost immediately runnable.
Don't forget to add System.Net.Http, specially if you receive this error:
Severity Code Description Project File Line Suppression State
Error CS0246 The type or namespace name 'HttpClient' could not be
found (are you missing a using directive or an assembly
reference?) 1_default.aspx D:\Projetos\Testes\FacebookAPI\FB-CustomAudience\default.aspx.cs 56 Active
In this case you shoud:
Add System.Net.Http from Nuget: Tools / NuGet Package Manager / Manager NuGet Packages for Solution;
Search for System.Net.Http
Add in the top of your page the follow code:
using System.Net.Http;
Call cURL from your console app is not a good idea.
But you can use TinyRestClient which make easier to build requests :
var client = new TinyRestClient(new HttpClient(),"https://api.repustate.com/");
client.PostRequest("v2/demokey/score.json").
AddQueryParameter("text", "").
ExecuteAsync<MyResponse>();
Well if you are new to C# with cmd-line exp. you can use online sites like "https://curl.olsh.me/" or search curl to C# converter will returns site that could do that for you.
or if you are using postman you can use Generate Code Snippet only problem with Postman code generator is the dependency on RestSharp library.
I want to make the following curl call in my C# console application:
curl -d "text=This is a block of text" \
http://api.repustate.com/v2/demokey/score.json
I tried to do like the question posted here, but I cannot fill the properties properly.
I also tried to convert it to a regular HTTP request:
http://api.repustate.com/v2/demokey/score.json?text="This%20is%20a%20block%20of%20text"
Can I convert a cURL call to an HTTP request? If so, how? If not, how can I make the above cURL call from my C# console application properly?
Well, you wouldn't call cURL directly, rather, you'd use one of the following options:
HttpWebRequest/HttpWebResponse
WebClient
HttpClient (available from .NET 4.5 on)
I'd highly recommend using the HttpClient class, as it's engineered to be much better (from a usability standpoint) than the former two.
In your case, you would do this:
using System.Net.Http;
var client = new HttpClient();
// Create the HttpContent for the form to be posted.
var requestContent = new FormUrlEncodedContent(new [] {
new KeyValuePair<string, string>("text", "This is a block of text"),
});
// Get the response.
HttpResponseMessage response = await client.PostAsync(
"http://api.repustate.com/v2/demokey/score.json",
requestContent);
// Get the response content.
HttpContent responseContent = response.Content;
// Get the stream of the content.
using (var reader = new StreamReader(await responseContent.ReadAsStreamAsync()))
{
// Write the output.
Console.WriteLine(await reader.ReadToEndAsync());
}
Also note that the HttpClient class has much better support for handling different response types, and better support for asynchronous operations (and the cancellation of them) over the previously mentioned options.
Or in restSharp:
var client = new RestClient("https://example.com/?urlparam=true");
var request = new RestRequest(Method.POST);
request.AddHeader("content-type", "application/x-www-form-urlencoded");
request.AddHeader("cache-control", "no-cache");
request.AddHeader("header1", "headerval");
request.AddParameter("application/x-www-form-urlencoded", "bodykey=bodyval", ParameterType.RequestBody);
IRestResponse response = client.Execute(request);
Below is a working example code.
Please note you need to add a reference to Newtonsoft.Json.Linq
string url = "https://yourAPIurl";
WebRequest myReq = WebRequest.Create(url);
string credentials = "xxxxxxxxxxxxxxxxxxxxxxxx:yyyyyyyyyyyyyyyyyyyyyyyyyyyyyy";
CredentialCache mycache = new CredentialCache();
myReq.Headers["Authorization"] = "Basic " + Convert.ToBase64String(Encoding.ASCII.GetBytes(credentials));
WebResponse wr = myReq.GetResponse();
Stream receiveStream = wr.GetResponseStream();
StreamReader reader = new StreamReader(receiveStream, Encoding.UTF8);
string content = reader.ReadToEnd();
Console.WriteLine(content);
var json = "[" + content + "]"; // change this to array
var objects = JArray.Parse(json); // parse as array
foreach (JObject o in objects.Children<JObject>())
{
foreach (JProperty p in o.Properties())
{
string name = p.Name;
string value = p.Value.ToString();
Console.Write(name + ": " + value);
}
}
Console.ReadLine();
Reference: TheDeveloperBlog.com
Late response but this is what I ended up doing. If you want to run your curl commands very similarly as you run them on linux and you have windows 10 or latter do this:
public static string ExecuteCurl(string curlCommand, int timeoutInSeconds=60)
{
if (string.IsNullOrEmpty(curlCommand))
return "";
curlCommand = curlCommand.Trim();
// remove the curl keworkd
if (curlCommand.StartsWith("curl"))
{
curlCommand = curlCommand.Substring("curl".Length).Trim();
}
// this code only works on windows 10 or higher
{
curlCommand = curlCommand.Replace("--compressed", "");
// windows 10 should contain this file
var fullPath = System.IO.Path.Combine(Environment.SystemDirectory, "curl.exe");
if (System.IO.File.Exists(fullPath) == false)
{
if (Debugger.IsAttached) { Debugger.Break(); }
throw new Exception("Windows 10 or higher is required to run this application");
}
// on windows ' are not supported. For example: curl 'http://ublux.com' does not work and it needs to be replaced to curl "http://ublux.com"
List<string> parameters = new List<string>();
// separate parameters to escape quotes
try
{
Queue<char> q = new Queue<char>();
foreach (var c in curlCommand.ToCharArray())
{
q.Enqueue(c);
}
StringBuilder currentParameter = new StringBuilder();
void insertParameter()
{
var temp = currentParameter.ToString().Trim();
if (string.IsNullOrEmpty(temp) == false)
{
parameters.Add(temp);
}
currentParameter.Clear();
}
while (true)
{
if (q.Count == 0)
{
insertParameter();
break;
}
char x = q.Dequeue();
if (x == '\'')
{
insertParameter();
// add until we find last '
while (true)
{
x = q.Dequeue();
// if next 2 characetrs are \'
if (x == '\\' && q.Count > 0 && q.Peek() == '\'')
{
currentParameter.Append('\'');
q.Dequeue();
continue;
}
if (x == '\'')
{
insertParameter();
break;
}
currentParameter.Append(x);
}
}
else if (x == '"')
{
insertParameter();
// add until we find last "
while (true)
{
x = q.Dequeue();
// if next 2 characetrs are \"
if (x == '\\' && q.Count > 0 && q.Peek() == '"')
{
currentParameter.Append('"');
q.Dequeue();
continue;
}
if (x == '"')
{
insertParameter();
break;
}
currentParameter.Append(x);
}
}
else
{
currentParameter.Append(x);
}
}
}
catch
{
if (Debugger.IsAttached) { Debugger.Break(); }
throw new Exception("Invalid curl command");
}
StringBuilder finalCommand = new StringBuilder();
foreach (var p in parameters)
{
if (p.StartsWith("-"))
{
finalCommand.Append(p);
finalCommand.Append(" ");
continue;
}
var temp = p;
if (temp.Contains("\""))
{
temp = temp.Replace("\"", "\\\"");
}
if (temp.Contains("'"))
{
temp = temp.Replace("'", "\\'");
}
finalCommand.Append($"\"{temp}\"");
finalCommand.Append(" ");
}
using (var proc = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "curl.exe",
Arguments = finalCommand.ToString(),
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
WorkingDirectory = Environment.SystemDirectory
}
})
{
proc.Start();
proc.WaitForExit(timeoutInSeconds*1000);
return proc.StandardOutput.ReadToEnd();
}
}
}
The reason why the code is a little bit long is because windows will give you an error if you execute a single quote. In other words, the command curl 'https://google.com' will work on linux and it will not work on windows. Thanks to that method I created you can use single quotes and run your curl commands exactly as you run them on linux. This code also checks for escaping characters such as \' and \".
For example use this code as
var output = ExecuteCurl(#"curl 'https://google.com' -H 'Accept: application/json, text/javascript, */*; q=0.01'");
If you where to run that same string agains C:\Windows\System32\curl.exe it will not work because for some reason windows does not like single quotes.
I know this is a very old question but I post this solution in case it helps somebody. I recently met this problem and google led me here. The answer here helps me to understand the problem but there are still issues due to my parameter combination. What eventually solves my problem is curl to C# converter. It is a very powerful tool and supports most of the parameters for Curl. The code it generates is almost immediately runnable.
Don't forget to add System.Net.Http, specially if you receive this error:
Severity Code Description Project File Line Suppression State
Error CS0246 The type or namespace name 'HttpClient' could not be
found (are you missing a using directive or an assembly
reference?) 1_default.aspx D:\Projetos\Testes\FacebookAPI\FB-CustomAudience\default.aspx.cs 56 Active
In this case you shoud:
Add System.Net.Http from Nuget: Tools / NuGet Package Manager / Manager NuGet Packages for Solution;
Search for System.Net.Http
Add in the top of your page the follow code:
using System.Net.Http;
Call cURL from your console app is not a good idea.
But you can use TinyRestClient which make easier to build requests :
var client = new TinyRestClient(new HttpClient(),"https://api.repustate.com/");
client.PostRequest("v2/demokey/score.json").
AddQueryParameter("text", "").
ExecuteAsync<MyResponse>();
Well if you are new to C# with cmd-line exp. you can use online sites like "https://curl.olsh.me/" or search curl to C# converter will returns site that could do that for you.
or if you are using postman you can use Generate Code Snippet only problem with Postman code generator is the dependency on RestSharp library.
I am attempting to read a MySQL database from my C# project using the MySQL drivers for .net off the MySQL site.
Though I did a bit of research on this (including this), I am still flummoxed why this is happening. I later ran a spike and I still get the same error. (Prior to running this I populated the database with some default values.) Here's the spike code in toto.
class Program {
static void Main (string[] args) {
Console.WriteLine (GetUserAge ("john")); // o/p's -1
}
static int GetUserAge (string username) {
string sql = "select age from users where name=#username";
int val = -1;
try {
using (MySqlConnection cnn = GetConnectionForReading ()) {
cnn.Open ();
MySqlCommand myCommand = new MySqlCommand (sql, cnn);
myCommand.Parameters.AddWithValue ("#username", username);
using (MySqlDataReader reader = myCommand.ExecuteReader ()) {
DataTable dt = new DataTable ();
dt.Load (reader);
if (reader.Read ()) {
val = reader.GetInt32 (0);
}
}
}
} catch (Exception ex) {
Console.WriteLine (ex.Message);
} finally {
}
return val;
}
private static MySqlConnection GetConnectionForReading () {
string conStr = "Data Source=localhost;Database=MyTestDB;User ID=testuser;Password=password";
return new MySqlConnection (conStr);
}
}
The code above gives me the exception: "Invalid attempt to Read when reader is closed."
Later I modified the if-condition like so:
if (reader.HasRows && reader.Read ()) {
val = reader.GetInt32 (0);
}
And now the o/p is -1. (The data's in there in the table.) If for some reason the result set had zero rows, the reader should not have got into the if-block in the first place. I mean, the whole point of the Read() method is to check if there are any rows in the result set in the first place.
At my wit's end here... just cannot figure out where I'm going wrong.
Thank you for your help! :)
I think using DataTable.Load will "consume" the reader and, at the very least, position it at the end. It may even account for the closed connection (but I'm just guessing here). What if you remove that line? I don't think it makes any sense here.