I have a routine (written with the generous help of others here) that allows me to take a List objects, and using any number of properties in any order it dynamically builds a TreeView structure with a Count at each node. This dynamic ability is a firm user requirement.
So a source List of:
{Prop1 = "A", Prop2 = "I", Prop3 = "X"},
{Prop1 = "A", Prop2 = "J", Prop3 = "X"},
{Prop1 = "B", Prop2 = "I", Prop3 = "X"},
{Prop1 = "B", Prop2 = "I", Prop3 = "Y"},
{Prop1 = "C", Prop2 = "K", Prop3 = "Z"}
Gives the following when the Selection is by Prop1 by Prop3:
Total (5)
- A(2)
- - X(2)
- B(2)
- - X(1)
- - Y(1)
- C(1)
- - Z(1)
Functionally this works fine. However, the performance leaves a lot to be desired when the number of distinct values increases. For example - one particular run in a dataset with 5K objects and 1K distinct values in Prop1 will take several seconds.
Here is the routine:
public static class TreeBuilder
{
public static Dictionary<string, TreeItem> BuildTree<TSource>(List<TSource> source, List<string> columns)
{
return new Dictionary<string, TreeItem>()
{
{ "Total",
new TreeItem()
{
Key = "Total",
RawKey = "Total",
Count = source.Count,
Items = GroupBySelector<TSource>(source, columns, 0, "Total")
}
}
};
}
public static MethodInfo GetGenericMethod(this Type type, string name, Type[] genericTypeArgs, Type[] paramTypes)
{
foreach (MethodInfo method in type.GetMethods())
if (method.Name == name)
{
var pa = method.GetParameters();
if (pa.Length == paramTypes.Length)
{
var genericMethod = method.MakeGenericMethod(genericTypeArgs);
if (genericMethod.GetParameters().Select(p => p.ParameterType).SequenceEqual(paramTypes))
return genericMethod;
}
}
return null;
}
private static MethodInfo GetGroupByMethodStatically<TElement, TKey>()
=> typeof(Enumerable).GetGenericMethod("GroupBy", new[] { typeof(TElement), typeof(TKey) }, new[] { typeof(IEnumerable<TElement>), typeof(Func<TElement, TKey>) });
private static MethodInfo GetEnumerableMethod(string methodName, Type tElement, Type tTKey)
{
var tIElement = typeof(IEnumerable<>).MakeGenericType(tElement);
var tFunction = typeof(Func<,>).MakeGenericType(tElement, tTKey);
return typeof(Enumerable).GetGenericMethod(methodName, new[] { tElement, tTKey }, new[] { tIElement, tFunction });
}
private static MethodInfo GetEnumerableMethod(string methodName, Type tElement)
{
var tIELEMENT = typeof(IEnumerable<>).MakeGenericType(tElement);
return typeof(Enumerable).GetGenericMethod(methodName, new[] { tElement }, new[] { tIELEMENT });
}
public static Dictionary<string, TreeItem> GroupBySelector<TElement>(IEnumerable<TElement> source, IList<string> columnNames, int entry = 0, string key = "")
{
if (source == null) throw new ArgumentNullException(nameof(source));
List<string> columnParameters = columnNames[entry].Split('|').ToList();
string columnName = columnParameters[0];
if (columnName == null) throw new ArgumentNullException(nameof(columnName));
if (columnName.Length == 0) throw new ArgumentException(nameof(columnName));
int nextEntry = entry + 1;
var tElement = typeof(TElement);
var tIElement = typeof(IEnumerable<TElement>);
var keyParm = Expression.Parameter(tElement);
var prop = Expression.Property(keyParm, columnName);
var param = Expression.Parameter(tIElement, "p");
var groupByMethod = GetEnumerableMethod("GroupBy", tElement, prop.Type);
var groupByExpr = Expression.Lambda(prop, keyParm);
var bodyExprCall = Expression.Call(groupByMethod, param, groupByExpr);
var tSelectInput = typeof(IGrouping<,>).MakeGenericType(prop.Type, tElement);
var selectParam = Expression.Parameter(tSelectInput, "p");
var tKey = typeof(TreeItem).GetMember("Key").Single();
var tRawKey = typeof(TreeItem).GetMember("RawKey").Single();
var tCount = typeof(TreeItem).GetMember("Count").Single();
var tParentKey = typeof(TreeItem).GetMember("ParentKey").Single();
var tItems = typeof(TreeItem).GetMember("Items").Single();
Expression selectParamKey = Expression.Property(selectParam, "Key");
Expression selectParamRawKey = selectParamKey;
if (selectParamKey.Type != typeof(string))
{
var toStringMethod = selectParamKey.Type.GetMethod("ToString", Type.EmptyTypes);
selectParamKey = Expression.Call(selectParamKey, toStringMethod);
}
if (selectParamRawKey.Type != typeof(string))
{
var toStringMethod = selectParamRawKey.Type.GetMethod("ToString", Type.EmptyTypes);
selectParamRawKey = Expression.Call(selectParamRawKey, toStringMethod);
}
var countMethod = GetEnumerableMethod("Count", tElement);
var countMethodExpr = Expression.Call(countMethod, selectParam);
var concatFullKeyExpr = Expression.Call(typeof(string).GetMethod("Concat", new[] { typeof(string), typeof(string), typeof(string) }),
Expression.Constant(key),
Expression.Constant("|"),
selectParamRawKey);
var groupBySelectorMethod = GetGenericMethod(MethodBase.GetCurrentMethod().DeclaringType, "GroupBySelector", new[] { tElement }, new[] { tIElement, typeof(IList<string>), typeof(int), typeof(string) });
var groupBySelectorMethodExpr = Expression.Call(groupBySelectorMethod, selectParam, Expression.Constant(columnNames), Expression.Constant(nextEntry), concatFullKeyExpr);
var newMenuItemExpr = Expression.New(typeof(TreeItem));
var selectBodyExpr = Expression.MemberInit(newMenuItemExpr, new[] {
Expression.Bind(tKey, selectParamKey),
Expression.Bind(tRawKey, selectParamRawKey),
Expression.Bind(tParentKey, Expression.Constant(key) ),
Expression.Bind(tCount, countMethodExpr),
Expression.Bind(tItems, groupBySelectorMethodExpr)
});
var selectBodyExprLamba = Expression.Lambda(selectBodyExpr, selectParam);
var selectBodyLastExpr = Expression.MemberInit(newMenuItemExpr, new[] {
Expression.Bind(tKey, selectParamKey),
Expression.Bind(tRawKey, selectParamRawKey),
Expression.Bind(tParentKey, Expression.Constant(key) ),
Expression.Bind(tCount, countMethodExpr)
});
var selectBodyLastExprLambda = Expression.Lambda(selectBodyLastExpr, selectParam);
var selectMethod = GetEnumerableMethod("Select", tSelectInput, typeof(TreeItem));
bodyExprCall = Expression.Call(selectMethod, bodyExprCall, (nextEntry < columnNames.Count) ? selectBodyExprLamba : selectBodyLastExprLambda);
var selectParamout = Expression.Parameter(typeof(TreeItem), "o");
Expression selectParamKeyout = Expression.Property(selectParamout, "FullKey");
var selectParamKeyLambda = Expression.Lambda(selectParamKeyout, selectParamout);
var lmi = GetEnumerableMethod("ToDictionary", typeof(TreeItem), typeof(string));
bodyExprCall = Expression.Call(lmi, bodyExprCall, selectParamKeyLambda);
var returnFunc = Expression.Lambda<Func<IEnumerable<TElement>, Dictionary<string, TreeItem>>>(bodyExprCall, param).Compile();
return returnFunc(source);
}
}
The routine is used to take data from a DB table and convert it into a hierarchical structure for use in a WPF TreeView.
Dictionary<string, TreeItem> treeItems = new Dictionary<string, TreeItem>();
treeItems = TreeBuilder.BuildTree<IDBRecord>(DBService.GetDBRecordList(), PropertySortList);
Can anyone offer any advice on how to improve the performance of this routine? Or suggest any alternative way of achieving the same result in a more performant way?
Thanks
A couple of optimizations are possible. A lot of time is spent in the call to Compile and you are calling Compile for each key at each level in the tree, which is adding up to a lot of overhead, about 7 seconds on my tests of 5k items. I first changed the code to pull out all static Reflection that had fixed types, so it is only done once per program run. That only made a small difference, since building the Expression tree is not the main issue.
I then changed the method to separate building the Expression from compiling the Expression and calling the resultant lambda. This allowed me to modify the recursive call to the Expression builder to instead be an inline Invoke of a new lambda for the new level. Then I called compile once on the resulting Expression and executed it. The new version no longer takes the entry parameter, but it could be put back in.
This reduced the overall time from about 7.6 seconds to 0.14 seconds for around a 50x speedup. A test with all three properties resulted in a 280x speedup.
If it is still possible for repeated calls to the method, adding a cache would be of even more benefit, though a quick test only shows about 14% time savings, and in the hundredths of a second of real time.
static MemberInfo tKey = typeof(TreeItem).GetMember("Key").Single();
static MemberInfo tRawKey = typeof(TreeItem).GetMember("RawKey").Single();
static MemberInfo tCount = typeof(TreeItem).GetMember("Count").Single();
static MemberInfo tParentKey = typeof(TreeItem).GetMember("ParentKey").Single();
static MemberInfo tItems = typeof(TreeItem).GetMember("Items").Single();
// Concat(string, string, string)
static MethodInfo Concat3MI = ((Func<string, string, string, string>)String.Concat).Method;
// new TreeItem() { ... }
static NewExpression newMenuItemExpr = Expression.New(typeof(TreeItem));
// Enumerable.ToDictionary<TreeItem>(IEnumerable<TreeItem>, Func<TreeItem,string>)
static MethodInfo ToDictionaryMI = GetEnumerableMethod("ToDictionary", typeof(TreeItem), typeof(string));
static Expression<Func<IEnumerable<TElement>, Dictionary<string, TreeItem>>> BuildGroupBySelector<TElement>(IList<string> columnNames, int entry, Expression key) {
List<string> columnParameters = columnNames[entry].Split('|').ToList();
string columnName = columnParameters[0];
if (columnName == null) throw new ArgumentNullException(nameof(columnName));
if (columnName.Length == 0) throw new ArgumentException(nameof(columnName));
int nextEntry = entry + 1;
var tElement = typeof(TElement);
var tIElement = typeof(IEnumerable<TElement>);
// (TElement kp)
var keyParm = Expression.Parameter(tElement, "kp");
// kp.columnName
var prop = Expression.Property(keyParm, columnName);
// (IEnumerable<TElement> p)
var IEParam = Expression.Parameter(tIElement, "p");
// GroupBy<TElement>(IEnumerable<TElement>, Func<TElement, typeof(kp.columnName)>)
var groupByMethod = GetEnumerableMethod("GroupBy", tElement, prop.Type);
// kp => kp.columnName
var groupByExpr = Expression.Lambda(prop, keyParm);
// GroupBy(p, kp => kp.columnName)
var bodyExprCall = Expression.Call(groupByMethod, IEParam, groupByExpr);
// typeof(IGrouping<typeof(kp.columnName), TElement>)
var tSelectInput = typeof(IGrouping<,>).MakeGenericType(prop.Type, tElement);
// (IGrouping<typeof(kp.columnName), TElement> sp)
var selectParam = Expression.Parameter(tSelectInput, "sp");
// sp.Key
Expression selectParamKey = Expression.Property(selectParam, "Key");
Expression selectParamRawKey = selectParamKey;
if (selectParamKey.Type != typeof(string)) {
var toStringMethod = selectParamKey.Type.GetMethod("ToString", Type.EmptyTypes);
// sp.Key.ToString()
selectParamKey = Expression.Call(selectParamKey, toStringMethod);
selectParamRawKey = selectParamKey;
}
// Count<TElement>()
var countMethod = GetEnumerableMethod("Count", tElement);
// sp.Count()
var countMethodExpr = Expression.Call(countMethod, selectParam);
LambdaExpression selectBodyExprLamba;
if (nextEntry < columnNames.Count) {
// Concat(key, "|", sp.Key.ToString())
var concatFullKeyExpr = Expression.Call(Concat3MI, key, Expression.Constant("|"), selectParamRawKey);
// p# => p#.GroupBy().Select().ToDictionary()
var groupBySelectorLambdaExpr = BuildGroupBySelector<TElement>(columnNames, nextEntry, (Expression)concatFullKeyExpr);
// Invoke(p# => p#..., sp#)
var groupBySelectorInvokeExpr = Expression.Invoke(groupBySelectorLambdaExpr, selectParam);
var selectBodyExpr = Expression.MemberInit(newMenuItemExpr, new[] {
Expression.Bind(tKey, selectParamKey),
Expression.Bind(tRawKey, selectParamRawKey),
Expression.Bind(tParentKey, key ),
Expression.Bind(tCount, countMethodExpr),
Expression.Bind(tItems, groupBySelectorInvokeExpr)
});
// sp => new TreeItem { Key = sp.Key.ToString(), RawKey = sp.Key.ToString(), ParentKey = key, Count = sp.Count(), Items = Invoke(p# => p#..., sp)) }
selectBodyExprLamba = Expression.Lambda(selectBodyExpr, selectParam);
}
else { // Last Level
var selectBodyExpr = Expression.MemberInit(newMenuItemExpr, new[] {
Expression.Bind(tKey, selectParamKey),
Expression.Bind(tRawKey, selectParamRawKey),
Expression.Bind(tParentKey, key ),
Expression.Bind(tCount, countMethodExpr)
});
// sp => new TreeItem { Key = sp.Key.ToString(), RawKey = sp.Key.ToString(), ParentKey = key, Count = sp.Count() }
selectBodyExprLamba = Expression.Lambda(selectBodyExpr, selectParam);
}
// Enumerable.Select<IGrouping<typeof<kp.columnName>, TElement>>(IEnumerable<IGrouping<>>, Func<IGrouping<>, TreeItem>)
var selectMethod = GetEnumerableMethod("Select", tSelectInput, typeof(TreeItem));
// p.GroupBy(kp => kp => kp.columnName).Select(sp => ...)
bodyExprCall = Expression.Call(selectMethod, bodyExprCall, selectBodyExprLamba);
// (TreeItem o)
var selectParamout = Expression.Parameter(typeof(TreeItem), "o");
// o.FullKey
Expression selectParamKeyout = Expression.Property(selectParamout, "FullKey");
// o => o.FullKey
var selectParamKeyLambda = Expression.Lambda(selectParamKeyout, selectParamout);
// p.GroupBy(...).Select(...).ToDictionary(o => o.FullKey)
bodyExprCall = Expression.Call(ToDictionaryMI, bodyExprCall, selectParamKeyLambda);
// p => p.GroupBy(kp => kp => kp.columnName).Select(sp => ...).ToDictionary(o => o.FullKey)
return Expression.Lambda<Func<IEnumerable<TElement>, Dictionary<string, TreeItem>>>(bodyExprCall, IEParam);
}
public static Dictionary<string, TreeItem> GroupBySelector<TElement>(IEnumerable<TElement> source, IList<string> columnNames, string key = "") {
if (source == null) throw new ArgumentNullException(nameof(source));
// p => p.GroupBy(kp => kp => kp.columnName).Select(sp => ...).ToDictionary(o => o.FullKey)
var returnFunc = BuildGroupBySelector<TElement>(columnNames, 0, Expression.Constant(key)).Compile();
return returnFunc(source);
}
I was trying to pivot a list of records, then i came across this :
public static DataTable PivotTable<T, TColumn, TGroupRow, TRow, TData>(
this System.Collections.Generic.IEnumerable<T> list,
Func<T, TColumn> column,
Expression<Func<T, TRow>> row,
Func<IEnumerable<T>, TData> dataSelector)
{
DataTable table = new DataTable();
var rowName = ((MemberExpression)row.Body).Member.Name;
table.Columns.Add(new DataColumn(rowName));
var columns = list.Select(column).Distinct();
foreach (var col in columns)
table.Columns.Add(new DataColumn(col.ToString()));
var rows = list.GroupBy(row.Compile())
.Select(rowGroup => new
{
Key = rowGroup.Key,
Values = columns.GroupJoin(
rowGroup,
c => c,
r => column(r),
(c, columnGroup) => dataSelector(columnGroup))
});
foreach (var rowVal in rows)
{
var dataRow = table.NewRow();
var items = rowVal.Values.Cast<object>().ToList();
items.Insert(0, rowVal.Key);
dataRow.ItemArray = items.ToArray();
table.Rows.Add(dataRow);
}
return table;
}
This is some solution offered from this post http://techbrij.com/pivot-c-array-datatable-convert-column-to-row-linq
my client code would be like this
var _dataSourceMatrix = //Some logic that get the DataTable records
var _departmentList = //Some logic that get the DataTable records
var _joinDataSourceDepartmentList = from a in _dataSourceMatrix.AsEnumerable()
join b in _departmentList.AsEnumerable()
on a.Field<int>("EntityID") equals b.Field<int>("DepartmentID")
select new { EntityName = b.Field<string>("Name"), Period = a.Field<string>("Period"), Value = a.Field<double>("Value"), EntityID = a.Field<int>("EntityID") };
_dataSourceMatrix = _joinDataSourceDepartmentList .OrderBy(x => x.Period).PivotTable(x => x.Period, x => x.EntityName, x => x.Sum(y => y.Value));
However, this is just the 3 columns Fields : Period, EntityName and Value could be utilize. I wish to utilize the EntityID as well. It should be one of the key to group by the record with EntityName. Is there any way to modify the PivotTable function ?
The objective to be achieve should be like this :
EntityName Period1, Period2, Period3, EntityID
A Value1 Value2 Value3 1
B Value1 Value2 Value3 2
I have the below document schema.
{
"name":"Name",
"region":"New Jersey",
"address":"92 Something Rd",
"city":"Jersey City",
"state":"NJ",
"zipCode":"07302",
"country":"USA",
amenities":[
"Sauna",
"Locker",
"Shop"
],
"services":[
"Car Rental",
"Transportation"
]
}
I want with one call to the server to get all the documents matching any of the filter arguments where map 1-1 meaning "state" = "NJ" OR "city" = "Jersey City" but also when any value of the list contained to any document array child, example [ "Sauna", "Locker" ] ANY IN "amenities". And it should be an OR concatenation of all the possible filters.
Using the C# MongoDB Driver I came up with the below methods so far in a MongoRepository class but doesn't return the desired results.
public async Task<IEnumerable<T>> DocumentsMatchEqFieldValueAsync<T>(string collectionName,
IDictionary<string, string> fieldsValues = null,
IDictionary<string, IEnumerable<string>> fieldsWithEnumerableValues = null,
IEnumerable<ObjectId> ids = null)
{
var cursor = await GetEqAsyncCursor<T>(collectionName, fieldsValues, fieldsWithEnumerableValues, ids).ConfigureAwait(false);
return await cursor.ToListAsync().ConfigureAwait(false);
}
protected Task<IAsyncCursor<T>> GetEqAsyncCursor<T>(string collectionName,
IDictionary<string, string> fieldsValues = null,
IDictionary<string, IEnumerable<string>> fieldsWithEnumerableValues = null,
IEnumerable<ObjectId> ids = null)
{
var collection = GetCollection<T>(collectionName);
var builder = Builders<T>.Filter;
// Not sure if this is the correct way to initialize it because it seems adding an empty filter condition returning ALL document;
FilterDefinition<T> filter = new BsonDocument();
if (fieldsValues != null &&
fieldsValues.Any())
{
filter = filter | fieldsValues
.Select(p => builder.Eq(p.Key, p.Value))
.Aggregate((p1, p2) => p1 | p2);
}
if (fieldsWithEnumerableValues != null &&
fieldsWithEnumerableValues.Any())
{
filter = filter | fieldsWithEnumerableValues
.Select(p => builder.AnyEq(p.Key, p.Value))
.Aggregate((p1, p2) => p1 | p2);
}
if (ids != null &&
ids.Any())
{
filter = filter | ids
.Select(p => builder.Eq("_id", p))
.Aggregate((p1, p2) => p1 | p2);
}
return collection.FindAsync(filter);
}
I want it to be generic so the client can call the method like this.
public async Task should_return_any_records_matching_all_possible_criteria()
{
// Arrange
IDocumentRepository documentRepository = new MongoRepository(_mongoConnectionString, _mongoDatabase);
// Act
var documents = await documentRepository.DocumentsMatchEqFieldValueAsync<BsonDocument>(Courses,
fieldsValues: new Dictionary<string, string>
{
{ "state", "NJ" },
{ "city", "Jersey City" }
},
fieldsWithEnumerableValues: new Dictionary<string, IEnumerable<string>>
{
{ "services", new List<string> { "Car Rental", "Locker" } },
{ "amenities", new List<string> { "Sauna", "Shop" } }
});
// Assert
documents.ShouldNotBeEmpty();
}
I would expect documents that have "state" = "NJ" OR "city" = "Jersey City" OR "services" CONTAINS ANY OF "Car Rental", "Locker" OR "amenities" CONTAINS ANY OF "Sauna", "Shop".
I am posting below the method I ended up using after some research for the future help of anyone looking to do the same. I found how to query using regex here, write plain MongoDB queries and add them to the filter collection here, and how to debug the generated query here.
After having all this information and a little bit of experimentation using the Studio 3T client find below the method.
protected Task<IAsyncCursor<T>> GetEqAsyncCursor<T>(string collectionName,
IDictionary<string, string> fieldEqValue = null,
IDictionary<string, string> fieldContainsValue = null,
IDictionary<string, IEnumerable<string>> fieldEqValues = null,
IDictionary<string, IEnumerable<string>> fieldElemMatchInValues = null,
IEnumerable<ObjectId> ids = null)
{
var collection = GetCollection<T>(collectionName);
var builder = Builders<T>.Filter;
IList<FilterDefinition<T>> filters = new List<FilterDefinition<T>>();
if (fieldEqValue != null &&
fieldEqValue.Any())
{
filters.Add(fieldEqValue
.Select(p => builder.Eq(p.Key, p.Value))
.Aggregate((p1, p2) => p1 | p2));
}
if (fieldContainsValue != null &&
fieldContainsValue.Any())
{
filters.Add(fieldContainsValue
.Select(p => builder.Regex(p.Key, new BsonRegularExpression($".*{p.Value}.*", "i")))
.Aggregate((p1, p2) => p1 | p2));
}
if (fieldEqValues != null &&
fieldEqValues.Any())
{
foreach (var pair in fieldEqValues)
{
foreach (var value in pair.Value)
{
filters.Add(builder.Eq(pair.Key, value));
}
}
}
if (fieldElemMatchInValues != null &&
fieldElemMatchInValues.Any())
{
var baseQuery = "{ \"%key%\": { $elemMatch: { $in: [%values%] } } }";
foreach (var item in fieldElemMatchInValues)
{
var replaceKeyQuery = baseQuery.Replace("%key%", item.Key);
var bsonQuery = replaceKeyQuery.Replace("%values%",
item.Value
.Select(p => $"\"{p}\"")
.Aggregate((value1, value2) => $"{value1},
{value2}"));
var filter = BsonSerializer.Deserialize<BsonDocument>(bsonQuery);
filters.Add(filter);
}
}
if (ids != null &&
ids.Any())
{
filters.Add(ids
.Select(p => builder.Eq("_id", p))
.Aggregate((p1, p2) => p1 | p2));
}
var filterConcat = builder.Or(filters);
// Here's how you can debug the generated query
//var documentSerializer = BsonSerializer.SerializerRegistry.GetSerializer<T>();
//var renderedFilter = filterConcat.Render(documentSerializer, BsonSerializer.SerializerRegistry).ToString();
return collection.FindAsync(filterConcat);
}
I would like to turn linq result into columns from rows, the field names are user changeable so I need the function to be dynamic.
sample data
ID: 331 FieldName: "BusinessCategory" FieldContents: "Regulatory"
ID: 331 FieldName: "PriorityGroup" FieldContents: "Must Do"
ID: 332 FieldName: "BusinessCategory" FieldContents: "Financial"
ID: 332 FieldName: "PriorityGroup" FieldContents: "Should Do"
Turn it into (sample end output)
ID BusinessCategory PriorityGroup
331 Regulatory Must Do
332 Financial Should DO
Here is the code block to extract to fieldnames and contents from the database.
public static IEnumerable<InitProjectValues1> GetProgramInitiativeAttributesPart1(int id)
{
using (dpm db = new dpm())
{
string partit = (string)HttpContext.Current.Session["sitePart"];
var configrefs = from c in (
from e in db.Metrics
join j in db.ProgramLink on e.ProjectRef equals j.LinkedProject
where (j.ProjectRef == id) && e.PartitNo == partit
select new
{
FieldName = e.FieldName,
FieldContents = e.MetricValue,
ProjectRef = e.ProjectRef,
})
select new InitProjectValues1
{
ProjectRef = c.ProjectRef,
FieldName = c.FieldName,
FieldContents = c.FieldContents,
}; //somewhere here would be the code to cover this into a single row per ProjectRef number.
return configrefs.ToList();
}
}
Here is the data model.
public class InitProjectValues1
{
public int? ProjectRef { get; set; }
public string FieldName { get; set; }
public string FieldContents { get; set; }
}
I really don't know where to go from here, hoping someone can provide guidance / sample code
The kind of operation you need is called a pivot. You are effectively rotating the table around a unique productRef and changing the rows to columns.
You could try this which makes use of a dynamic object which you require for dynamic column generation.
var configrefs = from c in (
from e in db.Metrics
join j in db.ProgramLink on e.ProjectRef equals j.LinkedProject
where (j.ProjectRef == id) && e.PartitNo == partit
select new
{
FieldName = e.FieldName,
FieldContents = e.MetricValue,
ProjectRef = e.ProjectRef,
}).ToArray();
return configrefs.ToPivotArray(
i => i.FieldName,
i => i.ProjectRef,
items => items.Any() ? items.FirstOrDefault().FieldContents : null);
Private method to get dynamic object:
private static dynamic GetAnonymousObject(IEnumerable<string> columns, IEnumerable<object> values)
{
IDictionary<string, object> eo = new ExpandoObject() as IDictionary<string, object>;
int i;
for (i = 0; i < columns.Count(); i++)
{
eo.Add(columns.ElementAt<string>(i), values.ElementAt<object>(i));
}
return eo;
}
And the extension method
public static dynamic[] ToPivotArray<T, TColumn, TRow, TData>(
this IEnumerable<T> source,
Func<T, TColumn> columnSelector,
Expression<Func<T, TRow>> rowSelector,
Func<IEnumerable<T>, TData> dataSelector)
{
var arr = new List<object>();
var cols = new List<string>();
String rowName = ((MemberExpression)rowSelector.Body).Member.Name;
var columns = source.Select(columnSelector).Distinct();
cols =(new []{ rowName}).Concat(columns.Select(x=>x.ToString())).ToList();
var rows = source.GroupBy(rowSelector.Compile())
.Select(rowGroup => new
{
Key = rowGroup.Key,
Values = columns.GroupJoin(
rowGroup,
c => c,
r => columnSelector(r),
(c, columnGroup) => dataSelector(columnGroup))
}).ToArray();
foreach (var row in rows)
{
var items = row.Values.Cast<object>().ToList();
items.Insert(0, row.Key);
var obj = GetAnonymousObject(cols, items);
arr.Add(obj);
}
return arr.ToArray();
}
Modified the ToPivotArray extension to handle multiple column selectors (using an anonymous class as the column selector)
public static dynamic[] ToPivotArrayNew<T, TColumn, TRow, TData>(
this IEnumerable<T> source,
Func<T, TColumn> columnSelector,
Expression<Func<T, TRow>> rowSelector,
Func<IEnumerable<T>, TData> dataSelector)
{
var arr = new List<object>();
var cols = new List<string>();
List<string> rowNames = new List<string>();
bool isObjectSelector = false;
if (rowSelector.Body.GetType() == typeof(MemberExpression))
{
rowNames.Add(((MemberExpression)rowSelector.Body).Member.Name);
}
else if (rowSelector.Body.GetType() == typeof(NewExpression))
{
isObjectSelector = true;
((NewExpression)rowSelector.Body).Members.ToList().ForEach(m => rowNames.Add(m.Name));
}
var columns = source.Select(columnSelector).Distinct();
cols = rowNames.ToArray().Concat(columns.Select(x => x.ToString())).ToList();
var rows = source.GroupBy(rowSelector.Compile())
.Select(rowGroup => new
{
Key = rowGroup.Key,
Values = columns.GroupJoin(
rowGroup,
c => c,
r => columnSelector(r),
(c, columnGroup) => dataSelector(columnGroup))
}).ToArray();
foreach (var row in rows)
{
var items = row.Values.Cast<object>().ToList();
if (isObjectSelector)
{
for (int i = 0; i < rowNames.Count(); i++)
{
items.Insert(i, row.Key.GetType().GetProperty(rowNames[i]).GetValue(row.Key));
}
}
else
{
items.Insert(0, row.Key);
}
var obj = GetAnonymousObject(cols, items);
arr.Add(obj);
}
return arr.ToArray();
}
i wrote function
private Func<CategorizedPosts, bool> CompileExpression(IEnumerable<Category> categories)
{
Expression predicateBody;
ParameterExpression pe = Expression.Parameter(typeof(CategorizedPosts), "post");
Expression left = Expression.Property(pe, typeof(CategorizedPosts).GetProperty("CATEGORY_ID"));
Expression right = Expression.Constant(categories.ElementAt(0).ID);
Expression equal = Expression.Equal(left, right);
predicateBody = equal;
for (int i = 1, j = categories.Count() - 1; i < categories.Count(); ++i )
{
var category = categories.ElementAt(i);
//y => y.CATEGORY_ID == 1 || y.CATEGORY_ID == 2)
left = Expression.Property(pe, typeof(CategorizedPosts).GetProperty("CATEGORY_ID"));
right = Expression.Constant(category.ID);
equal = Expression.Equal(left, right);
predicateBody = Expression.OrElse(predicateBody, equal);
}
var lll = Expression.Lambda<Func<CategorizedPosts, bool>>(predicateBody, pe);
var compiled = lll.Compile();
return compiled;
}
it compiles OK, but when I try to run this query
var ctx = db.Posts.Where(x => true);
if(predicate != null)
{
ctx = ctx.Where(x => x.CategorizedPosts.Where(**predicate**).Count() > 0);
}
IList<Post> posts = ctx.OrderByDescending(x => x.CREATION_DATE).Skip((page - 1) * perPage).Take(perPage).Select(x => new Post
{
POST_ID = x.ID,
TYPE = new Type { ID = x.TYPE_ID, NAME = x.Types.NAME },
AUTHOR = new Author()
{
ID = x.AUTHOR_ID,
NAME = x.Authors.NAME,
},
CATEGORIES = x.CategorizedPosts.Select(y => new Category() { ID = y.CATEGORY_ID, NAME = y.Categories.NAME }),
CREATION_DATE = x.CREATION_DATE,
}).ToList();
EF throws exception about internal error 1025 for Entity Data Provider. How can I perform this query with dynamic where?
You could use the Contains of a collection of Ids (int) and apply it on a where, for sample:
int[] categorieIds = categories.Select(x => x.Id).ToArray();
ctx = ctx.Where(x => x.CategorizedPosts.Any(c => categorieIds .Contains(c.Id));
Some Tips
Remember the Entity Framework works with Expression<Func<T, bool>> in the Where method, not only Func<T, bool>.
You also could try to apply PredicateBuilder class which provides some extensions methods like Or, And, Not, so, you could try this:
var predicate = PredicateBuilder.False<Product>();
foreach (string keyword in keywords)
{
string temp = keyword;
predicate = predicate.Or (p => p.Description.Contains (temp));
}
return dataContext.Products.Where(predicate).ToList();