I have a table in SQL Server that is populated from a Web Service. I want it to be refreshed on a scheduled basis. I would like to have something similar to an SQL Merge operation.
That is i define my source (Web Service) and my target (SQL Table) and i define how to handle missing from source missing from target and matches.
Lets consider a scenario where i have only two fields in the table Description and Deleted and the Web Service provides only the Description.
If a description is present in both the table and the web service then i just updated (or not).
If a description is present in the web service but not in the table i want it to be inserted
If a description is no longer present in the web server i want it marked as Deleted = true
What i currently have is:
public class WebServiceResults: AbstractOperation
{
public WebServiceResults()
{
var WebService = new WebService();
WSResults = WebService.GetResults();
}
IEnumerable<WSResult> WSResults { get; set; }
public override IEnumerable<Row> Execute(IEnumerable<Row> rows)
{
foreach(var obj in WSResults)
yield return Row.FromObject(obj);
}
}
class SQLTableResults : AbstractOperation
{
public SQLTableResults()
{
SQLResults = data.MyTable.Select(x=>new {x.Description,x.Deletet});
}
Data data = new Data();
IEnumerable<SQLResult> SQLResults { get; set; }
public override IEnumerable<Row> Execute(IEnumerable<Row> rows)
{
foreach (var obj in SQLResults)
yield return Row.FromObject(obj);
}
}
public override void Dispose()
{
data.Dispose();
base.Dispose();
}
}
class JoinTables : JoinOperation
{
protected override Row MergeRows(Row leftRow, Row rightRow)
{
Row row = leftRow.Clone();
row["Description2"] = rightRow["Description"];
return row;
}
protected override void SetupJoinConditions()
{
FullOuterJoin
.Left("Description")
.Right("Description");
}
}
class MergeTables : AbstractOperation
{
Data data = new Data();
public MergeTables()
{ }
public override IEnumerable<Row> Execute(IEnumerable<Row> rows)
{
foreach (var obj in rows)
{
if (String.IsNullOrEmpty((string)obj["Description2"]))
{
//Code for not matched at target
yield return Row.FromObject(obj);
}
if (String.IsNullOrEmpty((string)obj["Description"]))
{
//Code for not matched at source
yield return Row.FromObject(obj);
}
{
//Code for matched
yield return Row.FromObject(obj);
}
}
}
public override void Dispose()
{
data.Dispose();
base.Dispose();
}
}
protected override void Initialize()
{
Register(
new JoinTables()
.Left(new SQLTableResults())
.Right(new WebServiceResults())
);
Register(new MergeTables());
foreach (var error in GetAllErrors())
Console.Write(error.Message);
}
Is this the way to go? I would imagine something more of a stepped process, like
Register(new NotMatchedAtSourceOperation());
Register(new NotMatchedAtTargetOperation());
Register(new MatchedOperation());
but as i understand it, each register returns its rows to the next, so if i filter for the not matched, then the other two will do nothing.
Should i create a new process for each case?
By the way, i am looking for documentation on RhinoEtl. Do you know of any links? Any tutorials?
Determine the merge action in a single operation using a full outer join. You can see an example here. It's not exactly what you need, so I'll try to adapt it to your situation below:
protected override Row MergeRows(Row wsRow, Row dbRow) {
Row row;
// if the db row doesn't exist, then the ws row is new, and it should be inserted
if (dbRow["id"] == null) {
row = wsRow.Clone();
row["action"] = "Insert";
row["deleted"] = false;
return row;
}
// if the ws row doesn't exist, it should be marked as deleted in the database (if not already)
if (wsRow["id"] == null) {
row = dbRow.Clone();
row["deleted"] = true;
row["action"] = dbRow["deleted"].Equals(true) ? "None" : "Update";
return row;
}
// ws and db descriptions match, but check and make sure it's not marked as deleted in database
row = wsRow.Clone();
row["deleted"] = false;
row["action"] = dbRow["deleted"].Equals(true) ? "Update" : "None";
return row;
}
protected override void SetupJoinConditions() {
FullOuterJoin.Left("description").Right("description");
}
After you run this this operation, every row will have an action of "Insert", "Update", or "None." Based on this action, you can compose insert and update statements for a SqlBatchOperation to execute.
Related
Trying to use Except to exclude items from a list. However the following code is not working for me i.e. my list still includes the records that it should exclude. Is there anything obvious that I am doing wrong? Is there an issue with my loops? BTW the inverse of this code works i.e correct record is inserted when RunTime matches.
[HttpPost]
public JsonResult InsertActivities([FromBody] List<MemberData> customers)
{
var mData =_context.MemberData.Select(x => x.RunTime).ToList();
foreach (var item in mData)
{
var exclude = customers.Where(x => x.RunTime == item).ToList();
var list = customers.Except(exclude).ToList();
foreach (var data in list)
{
_context.MemberData.Add(data);
}
}
int insertedRecords = _context.SaveChanges();
return Json(insertedRecords);
}
Firstly be sure you have model design like below:
public class MemberData:IEquatable<MemberData>
{
public int Id { get; set; }
public string RunTime { get; set; }
public bool Equals(MemberData other)
{
if (other is null)
return false;
return this.RunTime == other.RunTime;
}
public override bool Equals(object obj) => Equals(obj as MemberData);
public override int GetHashCode() => (RunTime).GetHashCode();
}
Then change your code like below:
public JsonResult InsertActivities([FromBody] List<MemberData> customers)
{
//hard-coded the value...
//customers = new List<MemberData>()
//{
// new MemberData(){RunTime="aa"},
// new MemberData(){RunTime="bb"},
// new MemberData(){RunTime="ee"},
//}; //hard-coded the value...
var mData = _context.MemberData.ToList();
var list = customers.Except(mData);
foreach (var item in list)
{
foreach (var data in list)
{
_context.MemberData.Add(data);
}
}
int insertedRecords = _context.SaveChanges();
return Json(insertedRecords);
}
Note:
For inserting data to database successfully, if your model contains primary key, be sure the data's(after did Except operation) keys are not duplicated with the existing database data. Or you can just do like what I did in above code that do not set value for primary key.
What I want to achieve: I want to have a DataTable with rows of different types with different properties. One example is a sum row at the end of the DataTable (there will be more once this works).
Inspired by the two answers to this question describing extended DataRow classes (not the accepted one!) I have implemented the following:
public class ProjectEffortTable : DataTable
{
public ProjectEffortTable() : base() { }
public ProjectEffortTable(String TableName) : base(TableName) { }
protected ProjectEffortTable(System.Runtime.Serialization.SerializationInfo Info, System.Runtime.Serialization.StreamingContext Context) : base (Info, Context) { }
public ProjectEffortTable(String TableName, String TableNamespace) : base(TableName, TableNamespace) { }
protected override Type GetRowType()
{
return typeof(ProjectEffortRow);
}
protected override DataRow NewRowFromBuilder(DataRowBuilder Builder)
{
return new ProjectEffortRow(Builder);
}
}
public class ProjectEffortRow : DataRow
{
public ProjectEffortRow (DataRowBuilder Builder) : base (Builder)
{
}
public Boolean IsSum { get; set; }
}
With the following code I can include a new sum row:
var SumRow = ProjectEfforts.NewRow() as ProjectEffortRow;
SumRow.IsSum = true;
// calculate sums for all month columns
foreach (DataColumn Column in ProjectEfforts.Columns)
{
Decimal Sum = 0;
foreach (DataRow CurrentRow in ProjectEfforts.Rows)
{
if (CurrentRow[Column] is Double)
{
Sum += Convert.ToDecimal(CurrentRow[Column]);
}
}
SumRow[Column] = Decimal.Truncate(Sum);
}
ProjectEfforts.Rows.Add(SumRow);
The problem: The DataTable object can be manipulated by the user (using a DataGridView) and I need to save these changes to a data base in my data model (without saving the sum row).
To check for changes if have the following function:
Boolean CheckForChanges()
{
Boolean Changed = false;
var ProjectChanges = DataTableObject.GetChanges();
if (ProjectChanges != null)
{
for (var i = 0; i < ProjectChanges.Rows.Count; i++)
{
if (!(ProjectChanges.Rows[i] as ProjectEffortRow).IsSum)
{
Changed = true;
}
}
}
return Changed;
}
Unfortunately that method always returns true because it seems that GetChanges() creates a new DataTable where the information of the property is lost.
What I don't want to do: I don't want to add columns to the DataTable for each of the properties because this would tightly couple my view with the data model. If I created new columns for each property I would do that in the model and would need to hide all these columns in the view - which I deem quite ugly.
The question: Is it possible to somehow create a DataTable containing custom types of DataRows that maintain custom properties?
Thanks in advance for any help
After thinking some more I found a solution that works fine so far. I am not sure yet how well it scales but for the sum row I am quite satisfied. The key was to also implement GetChanges with custom code as the information about sum rows is known in that function.
Here's my current implementation:
public class ProjectEffortTable : DataTable
{
public ProjectEffortTable() : base() { }
public ProjectEffortTable(String TableName) : base(TableName) { }
protected ProjectEffortTable(System.Runtime.Serialization.SerializationInfo Info, System.Runtime.Serialization.StreamingContext Context) : base (Info, Context) { }
public ProjectEffortTable(String TableName, String TableNamespace) : base(TableName, TableNamespace) { }
protected override Type GetRowType()
{
return typeof(ProjectEffortRow);
}
protected override DataRow NewRowFromBuilder(DataRowBuilder Builder)
{
return new ProjectEffortRow(Builder);
}
public new ProjectEffortTable GetChanges()
{
var Changes = Clone() as ProjectEffortTable;
foreach (ProjectEffortRow CurrentRow in Rows)
{
if ((CurrentRow.RowState != DataRowState.Unchanged) && (!CurrentRow.IsSum))
{
Changes.ImportRow(CurrentRow);
}
}
if (Changes.Rows.Count == 0)
{
Changes = null;
}
return Changes;
}
public new ProjectEffortTable GetChanges(DataRowState RowStates)
{
var Changes = Clone() as ProjectEffortTable;
foreach (ProjectEffortRow CurrentRow in Rows)
{
if ((CurrentRow.RowState == RowStates) && (!CurrentRow.IsSum))
{
Changes.ImportRow(CurrentRow);
}
}
if (Changes.Rows.Count == 0)
{
Changes = null;
}
return Changes;
}
public void AddSumRow()
{
// add line with sum for each month column
var SumRow = NewRow() as ProjectEffortRow;
SumRow.IsSum = true;
Rows.Add(SumRow);
RecalculateSums();
}
public Boolean HasSumRow()
{
var SumRowFound = false;
if ((Rows[Rows.Count - 1] as ProjectEffortRow).IsSum)
{
SumRowFound = true;
}
return SumRowFound;
}
public void RemoveSumRow()
{
if (HasSumRow())
{
Rows[Rows.Count - 1].Delete();
}
}
private void RecalculateSums()
{
if (!HasSumRow())
{
throw new ApplicationException("Recalculation of sum triggered without sum row being present");
}
foreach (DataColumn Column in Columns)
{
Decimal Sum = 0;
foreach (ProjectEffortRow CurrentRow in Rows)
{
if ((CurrentRow[Column] is Double) && (!CurrentRow.IsSum))
{
Sum += Convert.ToDecimal(CurrentRow[Column]);
}
}
Rows[Rows.Count - 1][Column] = Decimal.Truncate(Sum);
}
}
}
I am trying the below way to return the dynamic results using dapper and stored procedure. Am I doing it in correct way?
using (IDbConnection dbConnection = Connection)
{
dbConnection.Open();
var result = dbConnection.Query<dynamic>("LMSSP_GetSelectedTableData",
new
{
TableName = TableName,
LangaugeID = AppTenant.SelectedLanguageID,
UserID = AppTenant.UserID
}, commandType: CommandType.StoredProcedure).ToList();
if (result != null)
{
// Added just for checking the data
foreach (var item in (IDictionary<string, object>)result.FirstOrDefault())
{
string key = item.Key;
string value = item.Value.ToString();
}
}
}
What my stored procedure do is, I will pass any table name and based on that it will return the results/records.So, obviously my number of records, columns will be varied as per the table name passed.
To achieve this I have used dynamic keyword along with dapper.
So my question is how can I pass this data to view as a model and render the controls on the view as per the properties/column data type. Can I get the data type of column OR PropertyInfo?
But, when dapper retrieves the records from database it returns as dapper row type?
Using same SP to fetch data from different table would be confusing (not good design). However to solve your problem technically, you can create model having list of control information. Example of control information
public class ControlInformation
{
public string Name { get; set; }
public dynamic Value { get; set; }
public string ControlType { get; set; }
// Applicable for drop down or multi select
public string AllValues { get; set; }
}
Model will have list of ControlInformations
public List<ControlInformation> ControlInformations { get; set; }
View will render the controls (partial view based on control type) Ex: very basic case to render different view for int and another view for rest. I have 2 partial views "IntCtrl" and "StringCtrl".
#foreach (var item in Model.ControlInformations)
{
if (#item.ControlType == "System.Int32")
{
Html.RenderPartial("IntCtrl", item);
}
else
{
Html.RenderPartial("StringCtrl", item);
}
}
Hope this help.
Here we are calling method which returns Datatable :
public DataTable GetMTDReport(bool isDepot, int userId)
{
using (IDbConnection _connection = DapperConnection)
{
var parameters = new DynamicParameters();
parameters.Add("#IsDepot", isDepot);
parameters.Add("#userId", userId);
var res = this.ExecuteSP<dynamic>(SPNames.SSP_MTDReport, parameters);
return ToDataTable(res);
}
}
In this we can call stored procedures by calling our custom method "ExecuteSP" :
public virtual IEnumerable<TEntity> ExecuteSP<TEntity>(string spName, object parameters = null)
{
using (IDbConnection _connection = DapperConnection)
{
_connection.Open();
return _connection.Query<TEntity>(spName, parameters, commandTimeout:0 , commandType: CommandType.StoredProcedure);
}
}
and here is "DapperConnection" method to connect the database:
You can give connection string with key ["MainConnection"]
public class DataConnection
{
public IDbConnection DapperConnection
{
get
{
return new SqlConnection(ConfigurationManager.ConnectionStrings["MainConnection"].ToString());
}
}
}
And at last we call "ToDataTable" method to change our response in datatable . We will receive response in DapperRow from the database because we passsed dynamic type in stored procedure.
public DataTable ToDataTable(IEnumerable<dynamic> items)
{
if (items == null) return null;
var data = items.ToArray();
if (data.Length == 0) return null;
var dt = new DataTable();
foreach (var pair in ((IDictionary<string, object>)data[0]))
{
dt.Columns.Add(pair.Key, (pair.Value ?? string.Empty).GetType());
}
foreach (var d in data)
{
dt.Rows.Add(((IDictionary<string, object>)d).Values.ToArray());
}
return dt;
}
I have a web form with around 50 fields that is used for crud operations on an Oracle DB, I am using EF6.
Currently, I accomplish this like so:
private GENERIC_FTP_SEND GetFields()
{
GENERIC_FTP_SEND ftpPartner = new GENERIC_FTP_SEND();
//Contact Info
ftpPartner.FTP_LOOKUP_ID = FTP_LOOKUP_IDTB.Text;
ftpPartner.PARTNER_NAME = PARTNER_NAMETB.Text;
ftpPartner.REMEDY_QUEUE = REMEDY_QUEUETB.Text;
ftpPartner.PRIORITY = PRIORITYBtns.SelectedValue;
ftpPartner.CONTACT_EMAIL = CONTACT_EMAILTB.Text;
ftpPartner.CONTACT_NAME = CONTACT_NAMETB.Text;
ftpPartner.CONTACT_PHONE = CONTACT_PHONETB.Text;
...
}
where GENERIC_FTP_SEND is the name of the virtual DbSet in my Model.context.cs.
This works fine but is not reusable in the least. What I would like to accomplish is to have some code that allows me to iterate through the attributes of ftpPartner and compare them to the field id for a match. Something like this:
var n =0;
foreach (Control cntrl in ControlList){
if(cntrl.ID == ftpPartner[n]){
ftpPartner[n] = cntrl.Text;
}
n++;
}
In case you need/want to see it here is my Model.context.cs
public partial class Entities : DbContext{
public Entities(): base("name=Entities")
{
}
protected override void OnModelCreating(DbModelBuilder modelBuilder)
{
throw new UnintentionalCodeFirstException();
}
public virtual DbSet<GENERIC_FTP_SEND> GENERIC_FTP_SEND { get; set; }
}
I saw the question here but I am not sure how to implement that in my case.
Entity Framework 6: is there a way to iterate through a table without holding each row in memory
You can achieve that with reflection:
var type = typeof(GENERIC_FTP_SEND);
foreach (Control cntrl in ControlList){
Object value = null;
if (cntrl is TextBox){
value = (cntrl as TextBox).Text;
} else if (cntrl is GroupBox){
value = (cntrl as GroupBox).SelectedValue;
} //etc ...
PropertyInfo pInfo = type.GetProperty(cntrl.ID);
if (pInfo != null && value != null){
pInfo.SetValue(ftpPartner, value, null);
}
}
You can also use the Entity Framework context object as well to accomplish the same if you know you are going to insert.
var x = new GENERIC_FTP_SEND();
// Add it to your context immediately
ctx.GENERIC_FTP_SEND.Add(x);
// Then something along these lines
foreach (Control cntrl in ControlList)
{
ctx.Entry(x).Property(cntrl.Name).CurrentValue = ctrl.Text;
}
I have this method
Meeting is a class
Attendees is an ICollection in Meeting
Class
public partial class Meeting
{
public Meeting()
{
this.Attendees = new List<Attendees>();
}
public virtual ICollection<Attendees> Attendees{ get; set; }
[...]
Method Controller
private void RemoveRowsDuplicated(Meeting model)
{
if (model.Attendees != null)
{
foreach (var item in model.Attendees.GroupBy(x => x.UserName).Select(y => y.Last()))
{
context.Attendees.Remove(item);
}
}
}
The objective is remove duplicate Attendees with the same username in the table.
But the current method it deletes all records and keeps the duplicate
Where am I going wrong?
Correct version of your method will look like this:
private static void RemoveRowsDuplicated(Meeting model)
{
if (model.Attendees != null)
{
var duplicates = new List<Attendees>();
foreach (var item in model.Attendees.GroupBy(x => x.UserName).Where(x=>x.Count()>1))
{
duplicates.AddRange(item.Skip(1));
}
duplicates.ForEach(x=>context.Attendees.Remove(x));
}
}
You can try writing raw SQL and invoking via EF and return Attendees objects in a list.
var query = "Select * from Attendees group by username";
var attendeesList = dbContext.Database.SqlQuery<Attendees>(query).ToList<Attendees>();
As I can see you grouped elements by name and remove last item. So you remove unique elements.
Like this
private void RemoveRowsDuplicated(Meeting model)
{
if (model.Attendees != null)
{
var temporaryAtendees = new List<Attendees>();
foreach(var item in model.Attendees)
{
if (temporaryAtendees.Contains(item))
{
context.Attendees.Remove(item);
}
else
{
temporaryAtendees.Add(item);
}
}
}
}