So I am quickly learning the ways of C# (complete noob that inherited this problem); I have written up the following code which calls a web service that returns JSON that is not always well-formed. The mission here is to take the JSON string and break it into array segments which get inserted into an SQL table for further parsing and testing. I.e. if the return string was something like
{1234:{5678:{1:{"key":"val","key":"val"},{2:{"key":"val","key":"val"}}}}
then the rows would be:
{1234}
{5678}
{1:{"key":"val","key":"val"}
{2:{"key":"val","key":"val"}
This is .NET 3.0 and SQL Server 2008 R2 (Legacy stuff).
Here is my working code:
public partial class UserDefinedFunctions
{
[Microsoft.SqlServer.Server.SqlFunction(DataAccess =
DataAccessKind.Read)]
public static SqlString TestParse(SqlString uri, SqlString username, SqlString passwd, SqlString postdata)
{
//-----
// The SqlPipe is how we send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
SqlString document;
try
{
// Set up the request, including authentication
WebRequest req = WebRequest.Create(Convert.ToString(uri));
if (Convert.ToString(username) != null & Convert.ToString(username) != "")
{
req.Credentials = new NetworkCredential(
Convert.ToString(username),
Convert.ToString(passwd));
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
//SqlContext.Pipe.Send("...get the data");
using (StreamReader rdr = new StreamReader(dataStream))
{
document = (SqlString)rdr.ReadToEnd();
rdr.Close();
//-----
string connectionString = null;
string sql = null;
connectionString = "Data source= 192.168.0.5; Database=Administration;User Id=Foo;Password=Blah; Trusted_Connection=True;";
using (SqlConnection cnn = new SqlConnection(connectionString))
{
sql = "INSERT INTO JSON_DATA (JSONROW) VALUES(#data)";
cnn.Open();
using (SqlCommand cmd = new SqlCommand(sql, cnn))
{
String payload = "";
String nestpayload = "";
int nests = 0;
String json = document.ToString();
/*first lets do some housekeeping on our payload; double closing curly braces need to be escaped (with curly braces!) in order to keep them in the string.*/
json = json.Replace("\\", "");
int i = json.Length;
//return new SqlString(json);
while (i > 1)
{
/*find the first closing "}" in the string and then check to see if there are more than one.
We need to read the data up to each closing brace, pull off that substring and process it for each iteration until the string is gone.*/
int closingbrace = json.IndexOf("}"); //First closing brace
int nextbrace = Math.Max(0, json.IndexOf("{", closingbrace)); //Next opening brace
String ChkVal = json.Substring(closingbrace + 1, Math.Max(1, nextbrace - closingbrace)); //+1 to ignore the 1st closing brace
int checks = Math.Max(0, ChkVal.Length) - Math.Max(0, ChkVal.Replace("}", "").Length);
payload = json.Substring(0, Math.Max(0, (json.IndexOf("}") + 1)));
/*Remove the payload from the string*/
json = json.Substring(payload.Length + 1);
/*"nests" is how many nested levels excluding the opening brace for the closing brace we found.*/
nests = (payload.Length - payload.Replace("{", "").Length);
/*If we have more then one nest level check to see if any of them go with the payload*/
if (nests > 1)
{
/*Break out the nested section and remove it from the payload.*/
nestpayload = payload.Substring(0, payload.LastIndexOf("{"));
payload = payload.Substring(payload.LastIndexOf("{"), payload.Length - payload.LastIndexOf("{"));
while (nests > 1)
{
if (checks > 0) //# of right braces in payload equals number of left-side nests go with the payload
{
// payload = nestpayload.Substring(Math.Max(0, nestpayload.LastIndexOf("{")), Math.Max(0, nestpayload.Length) - Math.Max(0, (nestpayload.LastIndexOf("{")))) + payload;//The second Math.Max defaults to 1; if we got here there is at minimum one "{" character in the substring
payload = nestpayload.Substring(nestpayload.LastIndexOf("{")) + payload;
nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0, nestpayload.LastIndexOf("{"))));
checks--;
nests--;
}
else
{
/*If we got here there are no more pieces of the nested data to append to the payload.
We use an array and string.split to keep the nest ordering correct.*/
string[] OrderedNest = nestpayload.Split('{');
for (int s = 0; s < OrderedNest.Length; s++)
{
if (OrderedNest[s] != "")
{
cmd.Parameters.AddWithValue("#data", "{" + OrderedNest[s].Replace(":", "}"));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
}
//cmd.Parameters.AddWithValue("#data", nestpayload.Substring(Math.Max(0,nestpayload.LastIndexOf("{"))).Replace(":","}"));
//cmd.Parameters.AddWithValue("#data", OrderedNest[1].Replace(":","}")+OrderedNest[2]);
// cmd.ExecuteNonQuery();
//cmd.Parameters.Clear();
//nests = Math.Max(0, nests - 1);
nests = 0;
//nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0,nestpayload.LastIndexOf("{"))));
}
}
}
/*At the very end payload will be a single "}"; check for this and discard the last row*/
if (payload != "}")
{
cmd.Parameters.AddWithValue("#data", new SqlChars(payload));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
/*Get the new string length*/
i = json.Length;
payload = "";
}
}
}
//-----
/* }
catch (Exception e)
{
return e.ToString();
}*/
}
// Close up everything...
dataStream.Close();
}
resp.Close();
// .. and return the output to the caller.
}//end using
return ("Finished");
}
catch (WebException e)
{
throw e;
}
}
}
While it works, it is INCREDIBLY slow; 4+ minutes to write 1500 rows to the server. Once daily this will need to write ~60,000 records in; the rest of the time it will be maybe 100 records POSTED and returned (I haven't worked up the POST part yet). I'm sure there are plenty of things I am doing not-so-proper here that are causing issues, but I have absolutely no idea where to even begin. I was excited enough that I could get the right response out of this! Any ideas/thoughts/help/sympathy would be greatly appreciated.
There are several problems here, not the least of which is that it appears you have posted your "sa" password to these here public interwebs. Here are the code issues that I see:
While it is possible to do Web Services calls in SQLCLR, it is definitely an advanced topic, full of pitfalls. This is not something that should be undertaken by a novice / beginner in SQLCLR, which itself is already a nuanced subset of regular .NET programming.
Get rid of the SqlPipe line and the comment line above it. Functions do not pass data back to the caller via SqlPipe; that is for Stored Procedures.
You probably shouldn't be using WebRequest
document should be string, not SqlString. You never return document and only ever convert it back to string, so it should just be that.
Use HttpWebRequest instead of WebRequest. This way you won't have to occasionally cast it into HttpWebRequest.
Don't convert the SqlString input parameters into string (e.g. Convert.ToString(uri)). All Sql* types have a Value property that returns the value in the native .NET type. So instead just use uri.Value, and so on.
Don't check for NULL inputs via Convert.ToString(username) != null. All Sql* type have an IsNull property that you can check. So instead use !username.IsNull.
DO NOT do all of your text processing (especially processing that contacts another system to do row-by-row inserts) while keeping the remote HttpWebRequest connection open. The only thing you should be doing within the using (WebResponse resp = req.GetResponse()) is populating the document variable. Don't do any processing of the contents of document until you are outside of that outer-most using().
Don't do individual inserts (i.e. the while (i > 1) loop). They aren't even in a transaction. If you get an error in the middle of the document, you will have loaded partial data (unless that's ok for this process).
ALWAYS Schema-qualify database objects. Meaning, JSON_DATA should be dbo.JSON_DATA (or whatever Schema is being used if not dbo).
In your connectionString you have both Id/Password and Trusted_Connection. Don't use both as they are mutually exclusive options (if you have both, the Id/Password are ignored and only Trusted_Connection is used).
Please, please do not log in as sa or have your application log in as sa. That is just begging for a disaster.
Are you connecting to a different instance of SQL Server than this SQLCLR object is running on? If it is the same instance, you might be better off changing this into a SqlProcedure so that you can use Context_Connection=True; as the connection string. That is the in-process connection that attaches to the session that it is being called from.
Don't use Parameters.AddWithValue(). Bad idea. Create the SqlParameter with the specific, and appropriate, datatype. Then add is to the Parameters collection via Add().
There might be other issues, but these were the obvious ones. As I said in point #1, you might be in over your head here. Not trying to be negative, just trying to avoid another poor implementation of SQLCLR that often leads to negative views of this otherwise very useful feature. If you want to pursue this, then please do more research first into how SQLCLR works, best practices, etc. A good place to start is a series that I am writing on this topic on SQL Server Central: Stairway to SQLCLR.
Or, another option is to use the INET_GetWebPages SQLCLR TVF that is available in the Full version of the SQL# SQLCLR library (which I wrote). This option is not free, but it would allow you to simply install the web request piece and then you just need to parse the returned document separately in a SQLCLR scalar UDF (which is probably the best approach anyway, even if you do the web request function / stored procedure on your own). In fact, if you are inserting into a table in the same Instance of SQL Server, you can create a SQLCLR TVF for the document parser and pass each OrderedNest value back using yield return (to stream the results back) and use as follows:
DECLARE #JSON NVARCHAR(MAX);
SELECT #JSON = [content]
FROM SQL#.INET_GetWebPages(#uri, .....);
INSERT INTO dbo.JSON_DATA (JSONROW)
SELECT [column_name]
FROM dbo.MyBrokenJsonFixerUpper(#JSON);
Good luck!
I am marking this question answered as it has become clear that what was need is a rewrite and rethinking of my original script. #Solomon Rutzky upvoted for providing helpful information which pointed me to this conclusion. For those
interested here is the rewrite:
using System;
using System.Data;
using System.Data.SqlClient;
using System.Data.SqlTypes;
using Microsoft.SqlServer.Server;
using System.Collections;
using System.Globalization;
// Other things we need for WebRequest
using System.Net;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
public partial class StoredProcedures
{
[Microsoft.SqlServer.Server.SqlProcedure]
public static void ApiParser(SqlString uri, SqlString user, SqlString pwd, SqlString postd)
{
// Create an SqlPipe to send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
//Make sure we have a url to process
if (uri.IsNull || uri.Value.Trim() == string.Empty)
{
pipe.Send("uri cannot be empty");
return;
}
try
{
//Create our datatable and get the table structure from the database
DataTable table = new DataTable();
string connectionString = null;
//connectionString = "Data source= 192.168.0.5; Database=Administration; Trusted_Connection=True;";
connectionString = "Data Source=(localdb)\\ProjectsV12;Integrated Security=True;Connect Timeout=30;Encrypt=False;TrustServerCertificate=False;ApplicationIntent=ReadWrite;MultiSubnetFailover=False";
using (SqlConnection gts = new SqlConnection(connectionString))
{
gts.Open();
using (SqlDataAdapter adapter = new SqlDataAdapter("SELECT TOP 0 * FROM sp_WebSvcs.dbo.JSON_DATA", gts))
{
adapter.Fill(table);
}
}
// Send a message string back to the client.
pipe.Send("Beginning Api Call...");
String json = "";
// Set up the request, including authentication
WebRequest req = HttpWebRequest.Create(uri.Value);
if (!user.IsNull & user.Value != "")
{
req.Credentials = new NetworkCredential(user.Value, pwd.Value);
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
using (StreamReader rdr = new StreamReader(dataStream))
{
json = (String)rdr.ReadToEnd();
rdr.Close();
}
// Close up everything...
dataStream.Close();
}
resp.Close();
}//end using resp
pipe.Send("Api Call complete; Parsing returned data...");
int i = 0;
String h = "";
String l = "";
int s = 0;
int p = 0;
int b = 0;
int payload = 0;
foreach (string line in json.Split(new[] { "}," }, StringSplitOptions.None))
{
if (line != "")
{
l = line;
i = l.Replace("{", "").Length + 1;
p = l.LastIndexOf("{");
if (line.Length > i) //we find this at the beginning of a group of arrays
{
h = line.Substring(0, p - 1);
s = Math.Max(0, h.LastIndexOf("{"));
if (h.Length > s && s != 0)
/*We have a nested array that has more than one level.
*This should only occur at the beginning of new array group.
*Advance the payload counter and get the correct string from line.*/
{
payload++;
l = line.Substring(s, line.Length - s);
}
h = (s >= 0) ? h.Substring(0, s) : h;
//=============
/*At this point 'h' is a nest collection. Split and add to table.*/
string[] OrderedNest = h.Split('{');
for (int z = 0; z < OrderedNest.Length; z++)
{
if (OrderedNest[z] != "")
{
table.Rows.Add(payload, "{" + OrderedNest[z].Replace(":", "").Replace("[","").Replace("]","") + "}");
}
}
//=============
}
else
{
h = null;
}
//at this point the first character in the row should be a "{"; If not we need to add one.
if (l[0].ToString() != "{")
{
l = "{" + l;
}
if (l.Replace("{", "").Length != l.Replace("}", "").Length) //opening and closing braces don't match; match the closing to the opening
{
l = l.Replace("}", "");
b = l.Length - l.Replace("{", "").Length;
l = l + new String('}', b);
}
table.Rows.Add(payload, l.Replace("\\\"", "").Replace("\\", "").Replace("]","").Replace("[",""));
}
}
//====
using (SqlConnection cnn = new SqlConnection(connectionString))
{
cnn.Open();
using (SqlBulkCopy copy = new SqlBulkCopy(cnn))
{
copy.DestinationTableName = "sp_WebSvcs.dbo.JSON_DATA";
copy.WriteToServer(table);
}
}
//====
} //end try
catch (Exception e)
{
pipe.Send("We have a problem!");
throw new Exception("\n\n" + e.Message + "\n\n");
}
pipe.Send("Parsing complete");
}
}
Please find the source code below. I am not getting any error but
_oracleCommand.ExecuteNonQuery() is going infinite - debugger is not moving to next line. I am able to update the data using Toad. I am stuck with this.
public void UpdateNewResignationRequestInSynergy(string _employeeno, string _comment, string _changeby, string _changeddt, string _reason)
{
int rowsaffected = 0;
string returnStatus;
string _synquery = "";
DateTime dtDate;
_employeeno = "774647";
_comment = "contract eand";
dtDate = DateTime.Parse(_changeddt, System.Globalization.CultureInfo.CreateSpecificCulture("en-CA"));
_oracleCommand = new OracleCommand(_synquery, _synergyDb);
_synergyDb.Open();
_oracleCommand.CommandText = string.Format(#"update wipinfo.fms_resignation set str_comments = 'contract end' where STR_CONTRACTOR_ID = 774647");
_oracleCommand.CommandType = CommandType.Text;
try
{
_oracleCommand.ExecuteNonQuery();
}
catch(Exception ex)
{
}
_synergyDb.Close();
}
You have to stop the oracle server and start again( do not restart it will not help you). I solved this problem by doing this process.
I am writing an application to transfer data from a simulated OPC server to a MySql Database. I am using some of the libraries from OPC Foundation - OPCNETAPI.DLL, OPCNETAPI.COM.DLL.
I have the program reading from the server happily, but it seems as soon as I open a connection inside the loop of reading data, it loops about 5 times and then stops executing.. There are no exceptions thrown and I have tried for hours to work it out and miserably failed.
I've also changed VisualStudio to break when an exception is thrown, and it doesn't
Hopefully some one may know whats going on, but there is nothing obvious, hopefully the following code will spark ideas!
Windows Forms Code
public void readplc()
{
Opc.URL url = new Opc.URL("opcda://localhost/Matrikon.OPC.Simulation.1");
Opc.Da.Server server = null;
OpcCom.Factory fact = new OpcCom.Factory();
server = new Opc.Da.Server(fact, null);
try
{
server.Connect(url, new Opc.ConnectData(new System.Net.NetworkCredential()));
}
catch (Exception ecy)
{
}
// Create a group
Opc.Da.Subscription group;
Opc.Da.SubscriptionState groupState = new Opc.Da.SubscriptionState();
groupState.Name = "Group";
groupState.Active = true;
group = (Opc.Da.Subscription)server.CreateSubscription(groupState);
// add items to the group.
Opc.Da.Item[] items = new Opc.Da.Item[2];
items[0] = new Opc.Da.Item();
items[0].ItemName = "Random.Int1";
items[1] = new Opc.Da.Item();
items[1].ItemName = "Random.Time";
items = group.AddItems(items);
group.DataChanged += new Opc.Da.DataChangedEventHandler(OnTransactionCompleted);
}
public void OnTransactionCompleted(object group, object hReq, Opc.Da.ItemValueResult[] items)
{
for (int i = 0; i < items.GetLength(0); i++)
{
try
{
string a = items[i].Value.ToString();
string b = items[i].ItemName;
string c = items[i].Key;
MySqlConnection conn = new MySqlConnection("server=localhost;user id=localhost;password=localhost;database=localhost;pooling=false");
conn.Open();
conn.Close();
}
catch (Exception ec)
{
MessageBox.Show(ec.Message);
}
}
}
I have set break points throughout the code, stepped through everything possible and still nothing obvious that's causing the issue.
The following code gives me the error (I get it from the MessageBox.Show() in the catch block)
"Exception in PopulateBla() : There is a file sharing violation. A
different process might be using the file [,,,,,,]
CODE
using (SqlCeCommand cmd = new SqlCeCommand(SQL_GET_VENDOR_ITEMS, new SqlCeConnection(SQLCE_CONN_STR)))
{
cmd.Parameters.Add("#VendorID", SqlDbType.NVarChar, 10).Value = vendorId;
cmd.Parameters.Add("#VendorItemID", SqlDbType.NVarChar, 19).Value = vendorItemId;
try
{
cmd.Connection.Open();
using (SqlCeDataReader SQLCEReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
{
if (SQLCEReader.Read())
{
itemID = SQLCEReader.GetString(ITEMID_INDEX);
packSize = SQLCEReader.GetString(PACKSIZE_INDEX);
recordFound = true;
}
}
}
catch (SqlCeException err)
{
MessageBox.Show(string.Format("Exception in PopulateControlsIfVendorItemsFound: {0}\r\n", err.Message));//TODO: Remove
}
finally
{
if (cmd.Connection.State == ConnectionState.Open)
{
cmd.Connection.Close();
}
}
}
SQL_GET_VENDOR_ITEMS is my query string.
What file sharing problem could be happening here?
UPDATE
This is the kind of code that makes that sort of refactoring recommended by ctacke below difficult:
public void setINVQueryItemGroup( string ID )
{
try
{
dynSQL += " INNER JOIN td_item_group ON t_inv.id = td_item_group.id AND t_inv.pack_size = td_item_group.pack_size WHERE td_item_group.item_group_id = '" + ID + "'";
}
catch( Exception ex )
{
CCR.ExceptionHandler( ex, "InvFile.setINVQueryDept" );
}
}
A SQL statement is being appended to by means of a separate method, altering a global var (dynSQL) while possibly allowing for SQL Injection (depending on where/how ID is assigned). If that's not enough, any exception thrown could mislead the weary bughunter due to indicating it occurred in a different method (doubtless the victim of a careless copy-and-paste operation).
This is "Coding Horror"-worthy. How many best practices can you ignore in a scant few lines of code?
Here's another example:
string dynSQL = "SELECT * FROM purgatory WHERE vendor_item = '" + VendorItem + "' ";
if (vendor_id != "")
{
dynSQL += "AND vendor_id = '" + vendor_id + "' ";
}
It could be done by replacing the args with "?"s, but the code to then determine which/how many params to assign would be 42X uglier than Joe Garagiola's mean cleats.
I really like Chris' idea of using a single connection to your database. You could declare that global to your class like so:
public ClayShannonDatabaseClass
{
private SqlCeConnection m_openConnection;
public ClayShannonDatabaseClass()
{
m_openConnection = new SqlCeConnection();
m_openConnection.Open();
}
public void Dispose()
{
m_openConnection.Close();
m_openConnection.Dispose();
m_openConnection = null;
}
}
I'm guessing your code is crashing whenever you attempt to actually open the database.
To verify this, you could stick an integer value in the code to help you debug.
Example:
int debugStep = 0;
try
{
//cmd.Connection.Open(); (don't call this if you use m_openConnection)
debugStep = 1;
using (SqlCeDataReader SQLCEReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
{
debugStep = 2;
if (SQLCEReader.Read())
{
debugStep = 3;
itemID = SQLCEReader.GetString(ITEMID_INDEX);
debugStep = 4;
packSize = SQLCEReader.GetString(PACKSIZE_INDEX);
debugStep = 5;
recordFound = true;
}
}
}
catch (SqlCeException err)
{
string msg = string.Format("Exception in PopulateControlsIfVendorItemsFound: {0}\r\n", err.Message);
string ttl = string.Format("Debug Step: {0}", debugStep);
MessageBox.Show(msg, ttl); //TODO: Remove
}
// finally (don't call this if you use m_openConnection)
// {
// if (cmd.Connection.State == ConnectionState.Open)
// {
// cmd.Connection.Close();
// }
// }
I'm guessing your error is at Step 1.
Provided the file isn't marked read-only (you checked that, right?), then you have another process with a non-sharing lock on the file.
The isql.exe database browser that comes with SQL CE is a common culprit if it's running in the background.
Depending on your version of SQLCE, it's quite possible that another process has an open connection (can't recall what version started allowing multiple process connections), so if you have any other app in the background that has it open, that may be a problem too.
You're also using a boatload of connections to that database, and they don't always get cleaned up and released immediately up Dispose. I'd highly recommend building a simple connection manager class that keeps a single (or more like two) connections to the database and just reuses them for all operations.
I am attempting to read a MySQL database from my C# project using the MySQL drivers for .net off the MySQL site.
Though I did a bit of research on this (including this), I am still flummoxed why this is happening. I later ran a spike and I still get the same error. (Prior to running this I populated the database with some default values.) Here's the spike code in toto.
class Program {
static void Main (string[] args) {
Console.WriteLine (GetUserAge ("john")); // o/p's -1
}
static int GetUserAge (string username) {
string sql = "select age from users where name=#username";
int val = -1;
try {
using (MySqlConnection cnn = GetConnectionForReading ()) {
cnn.Open ();
MySqlCommand myCommand = new MySqlCommand (sql, cnn);
myCommand.Parameters.AddWithValue ("#username", username);
using (MySqlDataReader reader = myCommand.ExecuteReader ()) {
DataTable dt = new DataTable ();
dt.Load (reader);
if (reader.Read ()) {
val = reader.GetInt32 (0);
}
}
}
} catch (Exception ex) {
Console.WriteLine (ex.Message);
} finally {
}
return val;
}
private static MySqlConnection GetConnectionForReading () {
string conStr = "Data Source=localhost;Database=MyTestDB;User ID=testuser;Password=password";
return new MySqlConnection (conStr);
}
}
The code above gives me the exception: "Invalid attempt to Read when reader is closed."
Later I modified the if-condition like so:
if (reader.HasRows && reader.Read ()) {
val = reader.GetInt32 (0);
}
And now the o/p is -1. (The data's in there in the table.) If for some reason the result set had zero rows, the reader should not have got into the if-block in the first place. I mean, the whole point of the Read() method is to check if there are any rows in the result set in the first place.
At my wit's end here... just cannot figure out where I'm going wrong.
Thank you for your help! :)
I think using DataTable.Load will "consume" the reader and, at the very least, position it at the end. It may even account for the closed connection (but I'm just guessing here). What if you remove that line? I don't think it makes any sense here.