We Keep Coding

C# parse SQL statement to find all INSERT/UPDATE/DELETE tables used in stored procedures

As the title says my intention is to find all tables participating in either INSERT/UPDATE/DELETE statements and produce a structured format. So far this is what I've come up with -
void Main()
{
string DBName = "Blah";
string ServerName = #"(localdb)\MSSQLLocalDB";
Server s = new Server(ServerName);
Database db = s.Databases[DBName];
ConcurrentDictionary<string, SPAudit> list = new ConcurrentDictionary<string, SPAudit>();
var sps = db.StoredProcedures.Cast<StoredProcedure>()
.Where(x => x.ImplementationType == ImplementationType.TransactSql && x.Schema == "dbo")
.Select(x => new
{
x.Name,
Body = x.TextBody
}).ToList();
Parallel.ForEach(sps, item =>
{
try
{
ParseResult p = Parser.Parse(item.Body);
IEnumerable<SqlInsertStatement> insStats = null;
IEnumerable<SqlUpdateStatement> updStats = null;
IEnumerable<SqlDeleteStatement> delStats = null;
var listTask = new List<Task>();
listTask.Add(Task.Run(() =>
{
insStats = FindBatchCollection<SqlInsertStatement>(p.Script.Batches);
}));
listTask.Add(Task.Run(() =>
{
updStats = FindBatchCollection<SqlUpdateStatement>(p.Script.Batches);
}));
listTask.Add(Task.Run(() =>
{
delStats = FindBatchCollection<SqlDeleteStatement>(p.Script.Batches);
}));
Task.WaitAll(listTask.ToArray());
foreach (var ins in insStats)
{
var table = ins?.InsertSpecification?.Children?.FirstOrDefault();
if (table != null)
{
var tableName = table.Sql.Replace("dbo.", "").Replace("[", "").Replace("]", "");
if (!tableName.StartsWith("#"))
{
var ll = list.ContainsKey(item.Name) ? list[item.Name] : null;
if (ll == null)
{
ll = new SPAudit();
}
ll.InsertTable.Add(tableName);
list.AddOrUpdate(item.Name, ll, (key, old) => ll);
}
}
}
foreach (var ins in updStats)
{
var table = ins?.UpdateSpecification?.Children?.FirstOrDefault();
if (table != null)
{
var tableName = table.Sql.Replace("dbo.", "").Replace("[", "").Replace("]", "");
if (!tableName.StartsWith("#"))
{
var ll = list.ContainsKey(item.Name) ? list[item.Name] : null;
if (ll == null)
{
ll = new SPAudit();
}
ll.UpdateTable.Add(tableName);
list.AddOrUpdate(item.Name, ll, (key, old) => ll);
}
}
}
foreach (var ins in delStats)
{
var table = ins?.DeleteSpecification?.Children?.FirstOrDefault();
if (table != null)
{
var tableName = table.Sql.Replace("dbo.", "").Replace("[", "").Replace("]", "");
if (!tableName.StartsWith("#"))
{
var ll = list.ContainsKey(item.Name) ? list[item.Name] : null;
if (ll == null)
{
ll = new SPAudit();
}
ll.DeleteTable.Add(tableName);
list.AddOrUpdate(item.Name, ll, (key, old) => ll);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
});
}
IEnumerable<T> FindBatchCollection<T>(SqlBatchCollection coll) where T : SqlStatement
{
List<T> sts = new List<T>();
foreach (var item in coll)
{
sts.AddRange(FindStatement<T>(item.Children));
}
return sts;
}
IEnumerable<T> FindStatement<T>(IEnumerable<SqlCodeObject> objs) where T : SqlStatement
{
List<T> sts = new List<T>();
foreach (var item in objs)
{
if (item.GetType() == typeof(T))
{
sts.Add(item as T);
}
else
{
foreach (var sub in item.Children)
{
sts.AddRange(FindStatement<T>(item.Children));
}
}
}
return sts;
}
public class SPAudit
{
public HashSet<string> InsertTable { get; set; }
public HashSet<string> UpdateTable { get; set; }
public HashSet<string> DeleteTable { get; set; }
public SPAudit()
{
InsertTable = new HashSet<string>();
UpdateTable = new HashSet<string>();
DeleteTable = new HashSet<string>();
}
}
Now I'm facing two problems
First, its is taking hell lot of a time to complete, given that there are around 841 stored procedures in the database.
Second, if there are statements like the following the table name is not being captured properly, meaning that the table is being captured as w instead of SomeTable_1 or SomeTable_2.
CREATE PROCEDURE [dbo].[sp_blah]
#t SomeTableType READONLY
AS
DELETE w
FROM SomeTable_2 w
INNER JOIN (Select * from #t) t
ON w.SomeID = t.SomeID
DELETE w
FROM SomeTable_1 w
INNER JOIN (Select * from #t) t
ON w.SomeID = t.SomeID
RETURN 0
Any help would be greatly appreciated.
Edit
Using the following dll from this location C:\Program Files (x86)\Microsoft SQL Server\140\DTS\Tasks-
Microsoft.SqlServer.ConnectionInfo.dll
Microsoft.SqlServer.Management.SqlParser.dll
Microsoft.SqlServer.Smo.dll
Microsoft.SqlServer.SqlEnum.dll

Finally I got it to work like I wanted the output to look like using #dlatikay answer. I'm posting this here more for documentation purposes than anything else.
I'm using the following nuget packages -
https://www.nuget.org/packages/Microsoft.SqlServer.SqlManagementObjects/
https://www.nuget.org/packages/Microsoft.SqlServer.TransactSql.ScriptDom/
and removed all other local dependencies. I hope this helps someone out there.
void Main()
{
string DatabaseName = "Blah";
string ServerIP = #"(localdb)\MSSQLLocalDB";
List<string> ExcludeList = new List<string>()
{
"sp_upgraddiagrams",
"sp_helpdiagrams",
"sp_helpdiagramdefinition",
"sp_creatediagram",
"sp_renamediagram",
"sp_alterdiagram",
"sp_dropdiagram"
};
List<string> StringDataTypes = new List<string>()
{
"nvarchar",
"varchar",
"nchar",
"char",
};
Server s = new Server(ServerIP);
s.SetDefaultInitFields(typeof(StoredProcedure), "IsSystemObject");
Database db = s.Databases[DatabaseName];
Dictionary<string, SPAudit> AuditList = new Dictionary<string, SPAudit>();
var sps = db.StoredProcedures.Cast<StoredProcedure>()
.Where(x => x.ImplementationType == ImplementationType.TransactSql && x.Schema == "dbo" && !x.IsSystemObject)
.Select(x => new
{
x.Name,
Body = x.TextBody,
Parameters = x.Parameters.Cast<StoredProcedureParameter>().Select(t =>
new SPParam()
{
Name = t.Name,
DefaultValue = t.DefaultValue,
DataType = $"{t.DataType.Name}{(StringDataTypes.Contains(t.DataType.Name) ? $"({(t.DataType.MaximumLength > 0 ? Convert.ToString(t.DataType.MaximumLength) : "MAX")})" : "")}"
})
}).ToList();
foreach (var item in sps)
{
try
{
TSqlParser parser = new TSql140Parser(true, SqlEngineType.Standalone);
IList<ParseError> parseErrors;
TSqlFragment sqlFragment = parser.Parse(new StringReader(item.Body), out parseErrors);
sqlFragment.Accept(new OwnVisitor(ref AuditList, item.Name, item.Parameters));
}
catch (Exception ex)
{
//Handle exception
}
}
}
public class OwnVisitor : TSqlFragmentVisitor
{
private string spname;
private IEnumerable<SPParam> parameters;
private Dictionary<string, SPAudit> list;
public OwnVisitor(ref Dictionary<string, SPAudit> _list, string _name, IEnumerable<SPParam> _parameters)
{
list = _list;
spname = _name;
parameters = _parameters;
}
public override void ExplicitVisit(InsertStatement node)
{
NamedTableReference namedTableReference = node?.InsertSpecification?.Target as NamedTableReference;
if (namedTableReference != null)
{
string table = namedTableReference?.SchemaObject.BaseIdentifier?.Value;
if (!string.IsNullOrWhiteSpace(table) && !table.StartsWith("#"))
{
if (!list.ContainsKey(spname))
{
SPAudit ll = new SPAudit();
ll.InsertTable.Add(table);
ll.Parameters.AddRange(parameters);
list.Add(spname, ll);
}
else
{
SPAudit ll = list[spname];
ll.InsertTable.Add(table);
}
}
}
base.ExplicitVisit(node);
}
public override void ExplicitVisit(UpdateStatement node)
{
NamedTableReference namedTableReference;
if (node?.UpdateSpecification?.FromClause != null)
{
namedTableReference = node?.UpdateSpecification?.FromClause?.TableReferences[0] as NamedTableReference;
}
else
{
namedTableReference = node?.UpdateSpecification?.Target as NamedTableReference;
}
string table = namedTableReference?.SchemaObject.BaseIdentifier?.Value;
if (!string.IsNullOrWhiteSpace(table) && !table.StartsWith("#"))
{
if (!list.ContainsKey(spname))
{
SPAudit ll = new SPAudit();
ll.UpdateTable.Add(table);
ll.Parameters.AddRange(parameters);
list.Add(spname, ll);
}
else
{
SPAudit ll = list[spname];
ll.UpdateTable.Add(table);
}
}
base.ExplicitVisit(node);
}
public override void ExplicitVisit(DeleteStatement node)
{
NamedTableReference namedTableReference;
if (node?.DeleteSpecification?.FromClause != null)
{
namedTableReference = node?.DeleteSpecification?.FromClause?.TableReferences[0] as NamedTableReference;
}
else
{
namedTableReference = node?.DeleteSpecification?.Target as NamedTableReference;
}
if (namedTableReference != null)
{
string table = namedTableReference?.SchemaObject.BaseIdentifier?.Value;
if (!string.IsNullOrWhiteSpace(table) && !table.StartsWith("#"))
{
if (!list.ContainsKey(spname))
{
SPAudit ll = new SPAudit();
ll.DeleteTable.Add(table);
ll.Parameters.AddRange(parameters);
list.Add(spname, ll);
}
else
{
SPAudit ll = list[spname];
ll.DeleteTable.Add(table);
}
}
}
base.ExplicitVisit(node);
}
}
public class SPAudit
{
public HashSet<string> InsertTable { get; set; }
public HashSet<string> UpdateTable { get; set; }
public HashSet<string> DeleteTable { get; set; }
public List<SPParam> Parameters { get; set; }
public SPAudit()
{
InsertTable = new HashSet<string>();
UpdateTable = new HashSet<string>();
DeleteTable = new HashSet<string>();
Parameters = new List<SPParam>();
}
}
public class SPParam
{
public string Name { get; set; }
public string DefaultValue { get; set; }
public string DataType { get; set; }
}

The SMO model exposes elements of the syntax tree. So instead of assuming a token by position, as in
UpdateSpecification?.Children?.FirstOrDefault();
look up the corresponding property in the documentation. For the update clause, the target table (or updatable view) can occur in different positions. Take this syntax:
UPDATE tablename SET column=value WHERE conditions
which is represented as
var targettable = ins?.UpdateSpecification?.Target?.ScriptTokenStream?.FirstOrDefault()?.Text;
in the SMO model. Whereas, a syntax unique to tsql,
UPDATE t SET t.columnname=value FROM tablename t WHERE conditions
will have its list of tables in the FROM clause.
Regarding the other two DML statements you mentioned: DELETE is the same because they share a common base class, DeleteInsertSpecification (Target).
For INSERT, there is the Target as well, and if its InsertSource is of type SelectInsertSource, this may be based on any number of tables and views too.

You can use following SQL Query:
SELECT *
FROM sys.dm_sql_referenced_entities ('dbo.APSP_MySP', 'OBJECT');
It gives you all the tables, views, SPs impacted in the stored procedure.
is_selected or is_select_all are set to 1 for selected references
is_updated is set to 1 for updated references
As query is reading from pre-defined system tables, it runs fast
If you need information about the referred object use the referenced_id column value to find details
You can use it in 2 ways:
Call the above query in parallel for each stored procedure
Create another query/SP which will loop and run it for every stored procedure

Change Proc_1 to your procedure name
Refine PATINDEX matching to cater for the different possibilites
Modify to look at all procedures
Does not cater for tables in dynamic sql or passed as parameters
Look out for any issues with dm_sql_referenced_entities
SELECT
e.TableName,
p.name,
PATINDEX('%DELETE '+e.TableName+'%', p.definition) AS is_delete,
PATINDEX('%INSERT INTO '+e.TableName+'%', p.definition) AS is_insert,
PATINDEX('%UPDATE '+e.TableName+'%', p.definition) AS is_update
FROM
(
SELECT distinct referenced_entity_name AS TableName
FROM sys.dm_sql_referenced_entities ('dbo.Proc_1', 'OBJECT')
) e,
(
SELECT o.name, m.object_id, definition
FROM sys.objects o, sys.sql_modules m
WHERE o.name = 'Proc_1'
AND o.type='P'
AND m.object_id = o.object_id
) p

I would recommend you querying the syscomments SQL view. The performance will be much better.
select text from sys.syscomments where text like '%DELETE%'
You can work with the results in the SQL Query or fetch all the results and filter the data in C#.

How to call a Generic method with dynamic properties in C#

I have the a few methods that have similar signature and was trying to convert them into one generic one without the use of interfaces.
public List<MultiSelectDropdown> ConvertListOfJobStatusToDropdownListClickable(List<JobStatus> js) {
var list = new List<MultiSelectDropdown>();
if (js != null && js.Count >= 1) {
list = js.Select(item => new MultiSelectDropdown { Name = item.StatusValue, Value = item.id.ToString() }).ToList();
}
return list;
}
public List<MultiSelectDropdown> ConvertListOfCUsersToDropdownListClickable(List<cUser> users) {
var list = new List<MultiSelectDropdown>();
if (users != null && users.Count >= 1) {
list = users.Select(item => new MultiSelectDropdown { Name = item.User_Name, Value = item.Id.ToString() }).ToList();
}
return list;
}
This is what I would like to do; pass in a list with two properties.
List<MultiSelectDropdown> ddlForClientUsers = ConvertToMultiSelectDropdownList(listOfClientsForUser, n => n.Client_Id, v => v.Client);
List<MultiSelectDropdown> ddlForJobStatus = ConvertToMultiSelectDropdownList(listOfJobStatus, n => n.Id, v => v.JobName);
This is the method I have tried but not sure how to get item.propName and item.propValue to work.
I get "Cannot resolve" propName and propValue in the method below
Is this possible?
public List<MultiSelectDropdown> ConvertToMultiSelectDropdownList<T, TPropertyName, TPropertyValue>(List<T> listOfT, Func<T, TPropertyName> propName, Func<T, TPropertyValue> propValue) {
var list = new List<MultiSelectDropdown>();
if (listOfT != null && listOfT.Count >= 1) {
list = listOfT.Select(item => new MultiSelectDropdown { Name = item.propName, Value = item.propValue }).ToList();
}
return list;
}
public class MultiSelectDropdown {
public string Name { get; set; }
public string Value { get; set; }
public bool IsChecked { get; set; }
}

Because the properties of your MultiSelectDropdown are strings, your functions should return those as well. And to invoke the functions, you have to write them like propName(item) instead of item.propName - that is the property syntax, and you indicated you didn't want to use interfaces.
public List<MultiSelectDropdown> ConvertToMultiSelectDropdownList<T>(List<T> listOfT, Func<T, string> propName, Func<T, string> propValue) {
var list = new List<MultiSelectDropdown>();
if (listOfT != null && listOfT.Count >= 1) {
list = listOfT.Select(item => new MultiSelectDropdown { Name = propName(item), Value = propValue(item) }).ToList();
}
return list;
}

You are really close, with just a slight mistake. The line (reformatted to prevent scrolling):
list = listOfT.Select(item => new MultiSelectDropdown
{
Name = item.propName,
Value = item.propValue
}).ToList();
needs to be:
list = listOfT.Select(item => new MultiSelectDropdown
{
Name = propName(item),
Value = propValue(item)
}).ToList();

Select an attribute from object in a List inside another List

Can I perform a select using ternary operator to get an attribute from object inside a list?
Here is my model:
public class Xpto
{
public List<Son> Sons { get; set; }
}
public class Son
{
public string Names { get; set; }
}
And here i would like to get "Name" attribute for each son that i have:
var result = (from a in mylist
select new
{
sonsNames = a.Sons == null : <What should i put here?>
}).ToList<object>();
I've tried Sons.ToString() but it prints an object reference.
I would like to have a string list in "sonsNames" and each name separeted by a ','. Example: sonsName: 'george, john'.

what about this ?
//set up a collection
var xptos = new List<Xpto>()
{ new Xpto()
{ Sons = new List<Son>
{ new Son() { Names = "test1" },
new Son() { Names = "test2" }
}
},
new Xpto()
{
Sons = new List<Son> {
new Son() { Names = "test3" }
}
}};
//select the names
var names = xptos.SelectMany(r => r.Sons).Where(k => k.Names != null)
.Select(r => r.Names + ",") .ToList();
names.ForEach(n => Console.WriteLine(n));
Here's more info on SelectMany()

Search for similar values in 2 arrays

I have List<Data> where Data is:
class Data
{
public int Id {get;set;}
public string Content {get;set;}
}
From Server I am getting following object List<ServerData> where ServerData:
class ServerData
{
int sId {get;set;}
Other stuff...
string sContent
}
Using LINQ, How I can find all matches that have same iD==sId, content==SContent?

just use a join ?
var matches = from data in listData
join serverData in listServerData
on new {id = data.Id, content = data.Content} equals
new {id = serverData.sId, content = serverData.sContent}
select new {
<whatever you need>
}

You can use this code:
IList<Data> data = //...
IList<ServerData> serverData = //...
IEnumerable<Data> matches = data.Where(d =>
serverData.Any(sd => sd.sId == d.Id &&
sd.sContent == d.Content));

Schema dependent class to schemaless document using MongoDb and C#

Let us suppose we have a document to store our client which has fixed and extra fields.
So here goes our sample class for the client:
public class Client
{
public string Name{ get; set; }
public string Address{ get; set; }
public List<ExtraField> ExtraFields{ get; set; } //these fields are extra ones
}
In extra field class we have something like this:
public class ExtraField
{
public string Key{ get; set; }
public string Type { get; set; }
public string Value { get; set; }
}
If I use standard driver's behaviour for serialization I would get smth like this:
{{Name:VName, Address:VAddress, ExtraFields:[{Key:VKey,Type:VType,
Value:VValue},...]}, document2,...,documentn}
While I would like to have something like this:
{{Name:VName, Address:VAddress, VKey:VValue,...}, document2,...,documentn}
This would improve the search performance and is generally the point of document orientation.
How can I customize the serialization to such a way?

Here is the way I solved it (it works fine) and solved the issue.
using System;
using System.Collections.Generic;
using System.Linq; using System.Text;
using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using MongoDB.Bson.Serialization.Serializers;
namespace TestDataGeneration {
public class FieldsWrapper : IBsonSerializable
{
public List<DataFieldValue> DataFieldValues { get; set; }
public object Deserialize(MongoDB.Bson.IO.BsonReader bsonReader, Type nominalType, IBsonSerializationOptions options)
{
if (nominalType != typeof(FieldsWrapper)) throw new ArgumentException("Cannot deserialize anything but self");
var doc = BsonDocument.ReadFrom(bsonReader);
var list = new List<DataFieldValue>();
foreach (var name in doc.Names)
{
var val = doc[name];
if (val.IsString)
list.Add(new DataFieldValue {LocalIdentifier = name, Values = new List<string> {val.AsString}});
else if (val.IsBsonArray)
{
DataFieldValue df = new DataFieldValue {LocalIdentifier = name};
foreach (var elem in val.AsBsonArray)
{
df.Values.Add(elem.AsString);
}
list.Add(df);
}
}
return new FieldsWrapper {DataFieldValues = list};
}
public void Serialize(MongoDB.Bson.IO.BsonWriter bsonWriter, Type nominalType, IBsonSerializationOptions options)
{
if (nominalType != typeof (FieldsWrapper))
throw new ArgumentException("Cannot serialize anything but self");
bsonWriter.WriteStartDocument();
foreach (var dataFieldValue in DataFieldValues)
{
bsonWriter.WriteName(dataFieldValue.LocalIdentifier);
if (dataFieldValue.Values.Count != 1)
{
var list = new string[dataFieldValue.Values.Count];
for (int i = 0; i < dataFieldValue.Values.Count; i++)
list[i] = dataFieldValue.Values[i];
BsonSerializer.Serialize(bsonWriter, list);
}
else
{
BsonSerializer.Serialize(bsonWriter, dataFieldValue.Values[0]);
}
}
bsonWriter.WriteEndDocument();
}
} }

Essentially you just need to implement two methods yourself. First one to serialize an object as you want and second to deserialize an object from db to your Client class back:
1 Seialize client class:
public static BsonValue ToBson(Client client)
{
if (client == null)
return null;
var doc = new BsonDocument();
doc["Name"] = client.Name;
doc["Address"] = client.Address;
foreach (var f in client.ExtraFields)
{
var fieldValue = new BsonDocument();
fieldValue["Type"] = f.Type;
fieldValue["Value"] = f.Value;
doc[f.Key] = fieldValue;
}
return doc;
}
2 Deserialize client object:
public static Client FromBson(BsonValue bson)
{
if (bson == null || !bson.IsBsonDocument)
return null;
var doc = bson.AsBsonDocument;
var client = new Client
{
ExtraFields = new List<ExtraField>(),
Address = doc["Address"].AsString,
Name = doc["Name"].AsString
};
foreach (var name in doc.Names)
{
var val = doc[name];
if (val is BsonDocument)
{
var fieldDoc = val as BsonDocument;
var field = new ExtraField
{
Key = name,
Value = fieldDoc["Value"].AsString,
Type = fieldDoc["Type"].AsString
};
client.ExtraFields.Add(field);
}
}
return client;
}
3 Complete test example:
I've added above two method to your client class.
var server = MongoServer.Create("mongodb://localhost:27020");
var database = server.GetDatabase("SO");
var clients = database.GetCollection<Type>("clients");
var client = new Client() {Id = ObjectId.GenerateNewId().ToString()};
client.Name = "Andrew";
client.Address = "Address";
client.ExtraFields = new List<ExtraField>();
client.ExtraFields.Add(new ExtraField()
{
Key = "key1",
Type = "type1",
Value = "value1"
});
client.ExtraFields.Add(new ExtraField()
{
Key = "key2",
Type = "type2",
Value = "value2"
});
//When inseting/saving use ToBson to serialize client
clients.Insert(Client.ToBson(client));
//When reading back from the database use FromBson method:
var fromDb = Client.FromBson(clients.FindOneAs<BsonDocument>());
4 Data structure in a database:
{
"_id" : ObjectId("4e3a66679c66673e9c1da660"),
"Name" : "Andrew",
"Address" : "Address",
"key1" : {
"Type" : "type1",
"Value" : "value1"
},
"key2" : {
"Type" : "type2",
"Value" : "value2"
}
}
BTW: Take a look into serialization tutorial as well.