I compare the speed of Sql with ElasticSearch. I have 1.5 million data. But sql query runs faster than elastic search. I don't understand the problem.
Why is the word like query coming faster in sql?
My code for Sql
public static List<Sales> GetAllRecords(string itemType)
{
List<Sales> salesReports = new List<Sales>();
string sqlQuery = String.Format(#"SELECT * FROM dbo.Sales where Region like '%{0}%'", itemType);
using (SqlConnection connection = new SqlConnection(CONNECTION_STRING))
{
var result = connection.Query<Sales>(sqlQuery);
foreach (var item in result)
{
Sales global = new Sales()
{
Region = item.Region,
Country = item.Country,
Item_Type=item.Item_Type,
Order_Date=item.Order_Date,
Order_ID = item.Order_ID,
Order_Priority=item.Order_Priority,
Sales_Channel=item.Sales_Channel,
Ship_Date = item.Ship_Date,
Total_Cost=item.Total_Cost,
Total_Profit=item.Total_Profit,
Total_Revenue=item.Total_Revenue,
Units_Sold=item.Units_Sold,
Unit_Cost=item.Unit_Cost,
Unit_Price = item.Unit_Price
};
salesReports.Add(global);
}
return result.ToList();
}
}
My code for ElasticSearch.
I search the data that I indexed before with elasticsearch here.
public static List<Sales> ConfigureES(string inputText)
{
List<Sales> salesReports = new List<Sales>();
// 1. Connection URL's elastic search
var listOfUrls = new Uri[]
{
// here we can set multple connectionn URL's...
new Uri("http://localhost:9200/")
};
StaticConnectionPool connPool = new StaticConnectionPool(listOfUrls);
ConnectionSettings connSett = new ConnectionSettings(connPool);
ElasticClient eClient = new ElasticClient(connSett);
// var see = eClient.DeleteIndex(INDEX_NAME);
// check the connection health
var checkClusterHealth = eClient.ClusterHealth();
if (checkClusterHealth.ApiCall.Success && checkClusterHealth.IsValid)
{
// 2. check the index exist or not
var checkResult = eClient.IndexExists(INDEX_NAME);
if (!checkResult.Exists)
{
// Raise error to Index not avaliable
}
// Search particular text field
var searchResponse = eClient.Search<Sales>(s =>
s.Index(INDEX_NAME).From(0).Size(5000).Scroll("10m")
.Query(q => q.Match(m => m.Field(f => f.Region).Query(inputText))));
//var results = eClient.Scroll<Salesreport>("10m", searchResponse.ScrollId);
while (searchResponse.Documents.Any())
{
var res = searchResponse.Documents;
var sds = res.Cast<Sales>();
salesReports.AddRange(sds);
searchResponse = eClient.Scroll<Sales>("10m", searchResponse.ScrollId);
}
}
else
{
// fail log the exception further use
var exception = checkClusterHealth.OriginalException.ToString();
var debugException = checkClusterHealth.DebugInformation.ToString();
}
return salesReports;
}
where I index data to elasticsearch.
public static string CONNECTION_STRING = string.Empty;
public static string INDEX_NAME = "elastic";
public static string INDEX_TYPE = "report4";
private static ElasticClient eClient;
static void Main(string[] args)
{
try
{
// read the config file ...
var configuration = new ConfigurationBuilder()
.SetBasePath(#"C:\Users\Celal\Desktop\ElasticSearch-master\ElasticSearch-master\ElasticSearchBGPJob\ElasticSearchBGPJob\ElasticSearchBGPJob")
.AddJsonFile("appsettings.json", false)
.Build();
CONNECTION_STRING = configuration.GetSection("DefaultConnection").Value;
if (string.IsNullOrEmpty(CONNECTION_STRING))
throw new ArgumentException("No connection string in appsettings.json");
// 1. Connection URL's elastic search
var listOfUrls =
// here we can set multple connectionn URL's...
new Uri("http://localhost:9200/");
ConnectionSettings connSett = new ConnectionSettings(listOfUrls);
eClient = new ElasticClient(connSett);
// var see = eClient.DeleteIndex(INDEX_NAME);
var createIndexDescriptor = new CreateIndexDescriptor(INDEX_NAME).Mappings(ms => ms.Map<Sales>(m => m.AutoMap()));
// check the connection health
var checkClusterHealth = eClient.ClusterHealth();
if (checkClusterHealth.ApiCall.Success && checkClusterHealth.IsValid)
{
// 2. check the index exist or not
var checkResult = eClient.IndexExists(INDEX_NAME);
if (!checkResult.Exists)
{
var createIndexResponse = eClient.CreateIndex(createIndexDescriptor);
if (createIndexResponse.ApiCall.Success && createIndexResponse.IsValid)
{
// index is created successfully....
}
else
{
// fail log the exception further use
var exception = createIndexResponse.OriginalException.ToString();
var debugException = createIndexResponse.DebugInformation.ToString();
}
}
// 3. get the last documet id of index
var lastRecordResponse = eClient.Search<Sales>(s => s
.Index(INDEX_NAME)
.Type(INDEX_TYPE)
.From(0)
.Size(1).Sort(sr => sr.Descending(f => f.Order_ID)));
if (lastRecordResponse.ApiCall.Success && lastRecordResponse.IsValid)
{
Console.WriteLine("Start " + DateTime.Now);
long salesRecordId = 0;
var listofrecords = new List<Sales>();
if (lastRecordResponse.Documents.Count >= 1)
{
var obj = lastRecordResponse.Documents;
foreach (var item in obj)
{
salesRecordId = item.Order_ID;
}
listofrecords = GetAllRecords(salesRecordId);
}
else
{
listofrecords = GetAllRecords(salesRecordId);
}
Console.WriteLine("END " + DateTime.Now);
// Insert the data into document format corresponding index...
if (listofrecords.Count > 0)
{
Console.WriteLine("===== START========= " + DateTime.Now);
BulkInsertData(listofrecords, eClient).Wait();
Console.WriteLine("===== END========= " + DateTime.Now);
}
}
else
{
// fail log the exception further use
var exception = lastRecordResponse.OriginalException.ToString();
var debugException = lastRecordResponse.DebugInformation.ToString();
}
}
else
{
// fail log the exception further use
var exception = checkClusterHealth.OriginalException.ToString();
var debugException = checkClusterHealth.DebugInformation.ToString();
}
Console.WriteLine("Hello World!");
Console.ReadLine();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.ReadLine();
}
}
public static List<Sales> GetAllRecords(long LastSalesId)
{
List<Sales> salesReports = new List<Sales>();
string sqlQuery = String.Format(#"SELECT * FROM dbo.Sales where Order_ID > {0} ", LastSalesId);
using (SqlConnection connection = new SqlConnection(CONNECTION_STRING))
{
connection.Open();
using (SqlCommand command = new SqlCommand(sqlQuery, connection))
{
command.CommandTimeout = 1000;
using (SqlDataReader dataReader = command.ExecuteReader())
{
if (dataReader.HasRows)
{
while (dataReader.Read())
{
Sales global = new Sales()
{
Order_ID=Convert.ToInt32(dataReader["Order_ID"]),
Region=Convert.ToString(dataReader["Region"]),
Country = Convert.ToString(dataReader["Country"]),
Total_Cost = (decimal)Convert.ToDouble(dataReader["Total_Cost"]),
Total_Revenue = Convert.ToString(dataReader["Total_Revenue"]),
Item_Type = Convert.ToString(dataReader["Item_Type"])
};
salesReports.Add(global);
}
}
}
}
connection.Close();
}
return salesReports;
}
static async Task BulkInsertData(List<Sales> ListofData, ElasticClient Eclient)
{
try
{
var splitTheLargeList = ChunkBy(ListofData);
var test = splitTheLargeList.LastOrDefault();
foreach (var item in splitTheLargeList)
{
var bulkResponse = await Eclient.BulkAsync(b => b
.Index(INDEX_NAME)
// .Type(INDEX_TYPE)
.IndexMany(item));
if (bulkResponse.ApiCall.Success && bulkResponse.IsValid)
{
// success fully inserted...
}
else
{
// fail log the exception further use
var exception = bulkResponse.OriginalException.ToString();
var debugException = bulkResponse.DebugInformation.ToString();
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.InnerException.ToString());
}
}
public static List<List<T>> ChunkBy<T>(List<T> source, int chunkSize = 1000)
{
return source
.Select((x, i) => new { Index = i, Value = x })
.GroupBy(x => x.Index / chunkSize)
.Select(x => x.Select(v => v.Value).ToList())
.ToList();
}
public static IEnumerable<List<T>> SplitList<T>(List<T> ListofData, int listSize = 1000)
{
for (int i = 0; i < ListofData.Count; i += listSize)
{
yield return ListofData.GetRange(i, Math.Min(listSize, ListofData.Count - i));
}
}
Related
I need some help. I need to combine these two methods into one and I can't figure out how.
I tried with an IF statement (we checked if employeeMark is NullOrEmpty, we use the first logic and if it's not, the second logic) but it's just duplicated code in one method instead of two.
This is the first method :
private async Task<ProcessedContractsIdentifiers> GetContractsIdentifiersByContractsCodes(List<ImportableContractIdentifier> importableContractsIdentifiers)
{
var errors = new List<ContractImportValidationErrorDto>();
foreach (var contract in importableContractsIdentifiers.Where(contract => string.IsNullOrWhiteSpace(contract.ContractCode)))
{
errors.Add(new ContractImportValidationErrorDto
{
ValidatedCode = string.Empty,
Line = contract.RowNumber,
ErrorCode = ContractImportValidationErrorCode.ContractCodeIsRequired
});
}
var contractsCodes = importableContractsIdentifiers.ConvertAll(x => x.ContractCode);
contractsCodes = contractsCodes.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
var contractsSignatures = await GetContractsSignaturesByContractsCodes(contractsCodes);
var unresolvedContracts = contractsCodes.Except(contractsSignatures.Select(cs => cs.Code)).ToHashSet();
foreach (var c in unresolvedContracts)
{
errors.Add(new ContractImportValidationErrorDto
{
ValidatedCode = c,
ErrorCode = ContractImportValidationErrorCode.InvalidContractCode
});
}
return new ProcessedContractsIdentifiers
{
IdentificationMethod = ContractIdentificationMethod.ByContractCode,
IdentifiedContractsSignatures = contractsSignatures,
NonIdentifiableContracts = errors
};
}
And this is the second method :
private async Task<ProcessedContractsIdentifiers> GetContractsIdentifiersByEmployeesMarks(List<ImportableContractIdentifier> importableContractsIdentifiers)
{
const int withSingleContract = 1;
var errors = new List<ContractImportValidationErrorDto>();
foreach (var contract in importableContractsIdentifiers.Where(contract => string.IsNullOrWhiteSpace(contract.EmployeeMark)))
{
errors.Add(new ContractImportValidationErrorDto
{
ValidatedCode = string.Empty,
Line = contract.RowNumber,
ErrorCode = ContractImportValidationErrorCode.EmployeeMarkIsRequired
});
}
var employeesMarks = importableContractsIdentifiers.Select(x => x.EmployeeMark).Where(x => !string.IsNullOrWhiteSpace(x)).ToHashSet();
var contractsSignatures = await GetContractsSignaturesByEmployeeMark(employeesMarks);
var contractsGroupedByEmployeeMark = contractsSignatures.GroupBy(x => x.EmployeeMark);
foreach (var contracts in contractsGroupedByEmployeeMark.Where(x => x.ToList().Count != withSingleContract))
{
errors.Add(new ContractImportValidationErrorDto
{
ValidatedCode = contracts.Key,
ErrorCode = ContractImportValidationErrorCode.EmployeeMarkWithMultipleContracts
});
}
var unresolvedMarks = employeesMarks.Except(contractsSignatures.Select(cs => cs.EmployeeMark)).ToHashSet();
foreach (var eMark in unresolvedMarks)
{
errors.Add(new ContractImportValidationErrorDto
{
ValidatedCode = eMark,
ErrorCode = ContractImportValidationErrorCode.InvalidEmployeeMark
});
}
return new ProcessedContractsIdentifiers
{
IdentificationMethod = ContractIdentificationMethod.ByEmployeeMark,
IdentifiedContractsSignatures = contractsSignatures,
NonIdentifiableContracts = errors
};
}
Use Enum as parameter, so according to its value you can write code blocks.
public Enum IOMode
{
In=0,
Out=1
}
public void CalculateAttendance(List<Log> lstLog, IOMode eIOMode)
{
if(eIOMode==IOMode.In)
{}
else
{ }
}
call CalculateAttendance(lstLog, IOMode.In) etc
I am using HangFire to run the job. I have a method that retrieve data from more than 50 sql servers.
I want to use HangFire to run each location separately by LocationID, and if 1 location fails I want that job get running again for that location after x minutes.
Also I want to see the log of the job on the HangFire Dashboard.
My job method :
public void SyncCompInfo()
{
List<Location> locations;
using (var db = new CompInfoDemoEntities())
{
locations = db.Locations.Where(x => x.IsActive).ToList();
}
foreach (var location in locations)
{
try
{
using (var _db = new CompInfoDemoEntities())
{
_log.Info("Getting CompoInfo data from location: " + location.StoreName);
_syncLogService.Add("Getting CompoInfo data from location: " + location.StoreName);
var responses = new List<CompInfoResponse>();
var compInfos = _db.IMS_CompInfo.Where(x => x.LocationId == location.Id).ToList();
using (var cnn = new SqlConnection(location.ConnectionString))
{
var sql = "select * from IMS_CompInfo;";
var sqlCmd = new SqlCommand(sql, cnn);
cnn.Open();
using (SqlDataReader rdr = sqlCmd.ExecuteReader())
{
while (rdr.Read())
{
var item = new CompInfoResponse();
item.Id = int.Parse(rdr["Id"].ToString());
item.ClientID = rdr["ClientID"].ToString();
item.LicenceID = rdr["LicenceID"].ToString();
item.POSCode = rdr["POSCode"].ToString();
item.logiPOS_Version = rdr["logiPOS_Version"].ToString();
if (rdr["LastLoginDate"] != null)
{
item.LastLoginDate = DateTime.Parse(rdr["LastLoginDate"].ToString());
}
item.ComputerName = rdr["ComputerName"].ToString();
if (rdr["BootTime"] != null)
{
item.BootTime = DateTime.Parse(rdr["BootTime"].ToString());
}
item.Domain = rdr["Domain"].ToString();
item.Manufacturer = rdr["Manufacturer"].ToString();
item.Model = rdr["Model"].ToString();
item.Memory = rdr["Memory"].ToString();
item.OS = rdr["OS"].ToString();
item.Build = rdr["Build"].ToString();
item.CPU = rdr["CPU"].ToString();
item.ProcArchitecture = rdr["ProcArchitecture"].ToString();
item.IP1 = rdr["IP1"].ToString();
item.MAC1 = rdr["MAC1"].ToString();
if (rdr["LastModifiedDate"] != null)
{
item.LastModifiedDate = DateTime.Parse(rdr["LastModifiedDate"].ToString());
}
if (rdr["Tag"] != null)
{
item.Tag = int.Parse(rdr["Tag"].ToString());
}
item.Application = rdr["Application"].ToString();
responses.Add(item);
}
}
}
What you seem to need is something like this, with a method you call upon startup, which loops other the locations and enqueues a job for each location.
I over simplified the thing (for example making the methods static),
but I guess most of the idea is there.
Have a look at Hangfire recurring tasks I guess it may be better suited to your needs than tasks fired upon application startups.
public void CalledUponStartup()
{
List<Location> locations;
using (var db = new CompInfoDemoEntities())
{
locations = db.Locations.Where(x => x.IsActive).ToList();
}
foreach (var location in locations)
{
BackgroundJob.Enqueue(() => SyncCompInfo(location.Id));
}
}
public static void SyncCompInfo(int locationId)
{
try
{
using (var _db = new CompInfoDemoEntities())
{
var location = db.Locations.FirstOrDefault(x => x.Id == locationId);
_log.Info("Getting CompoInfo data from location: " + location.StoreName);
_syncLogService.Add("Getting CompoInfo data from location: " + location.StoreName);
var responses = new List<CompInfoResponse>();
var compInfos = _db.IMS_CompInfo.Where(x => x.LocationId == location.Id).ToList();
using (var cnn = new SqlConnection(location.ConnectionString))
{
var sql = "select * from IMS_CompInfo;";
var sqlCmd = new SqlCommand(sql, cnn);
cnn.Open();
using (SqlDataReader rdr = sqlCmd.ExecuteReader())
{
while (rdr.Read())
{
var item = new CompInfoResponse();
item.Id = int.Parse(rdr["Id"].ToString());
item.ClientID = rdr["ClientID"].ToString();
item.LicenceID = rdr["LicenceID"].ToString();
item.POSCode = rdr["POSCode"].ToString();
item.logiPOS_Version = rdr["logiPOS_Version"].ToString();
if (rdr["LastLoginDate"] != null)
{
item.LastLoginDate = DateTime.Parse(rdr["LastLoginDate"].ToString());
}
item.ComputerName = rdr["ComputerName"].ToString();
if (rdr["BootTime"] != null)
{
item.BootTime = DateTime.Parse(rdr["BootTime"].ToString());
}
item.Domain = rdr["Domain"].ToString();
item.Manufacturer = rdr["Manufacturer"].ToString();
item.Model = rdr["Model"].ToString();
item.Memory = rdr["Memory"].ToString();
item.OS = rdr["OS"].ToString();
item.Build = rdr["Build"].ToString();
item.CPU = rdr["CPU"].ToString();
item.ProcArchitecture = rdr["ProcArchitecture"].ToString();
item.IP1 = rdr["IP1"].ToString();
item.MAC1 = rdr["MAC1"].ToString();
if (rdr["LastModifiedDate"] != null)
{
item.LastModifiedDate = DateTime.Parse(rdr["LastModifiedDate"].ToString());
}
if (rdr["Tag"] != null)
{
item.Tag = int.Parse(rdr["Tag"].ToString());
}
item.Application = rdr["Application"].ToString();
responses.Add(item);
}
}
I am looking to adjust the template project to add an opening for the door so the door will be flush with the wall system instead of appearing embedded in the skin of the wall any help would be very appreciated.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Xbim.Common;
using Xbim.Common.Step21;
using Xbim.Ifc;
using Xbim.IO;
using Xbim.Ifc4.ActorResource;
using Xbim.Ifc4.DateTimeResource;
using Xbim.Ifc4.ExternalReferenceResource;
using Xbim.Ifc4.PresentationOrganizationResource;
using Xbim.Ifc4.GeometricConstraintResource;
using Xbim.Ifc4.GeometricModelResource;
using Xbim.Ifc4.GeometryResource;
using Xbim.Ifc4.Interfaces;
using Xbim.Ifc4.Kernel;
using Xbim.Ifc4.MaterialResource;
using Xbim.Ifc4.MeasureResource;
using Xbim.Ifc4.ProductExtension;
using Xbim.Ifc4.ProfileResource;
using Xbim.Ifc4.PropertyResource;
using Xbim.Ifc4.QuantityResource;
using Xbim.Ifc4.RepresentationResource;
using Xbim.Ifc4.SharedBldgElements;
namespace HelloWall
{
class Program
{
/// <summary>
/// This sample demonstrates the minimum steps to create a compliant IFC model that contains a single standard case wall
/// </summary>
static int Main()
{
//first create and initialise a model called Hello Wall
Console.WriteLine("Initialising the IFC Project....");
using (var model = CreateandInitModel("HelloWall"))
{
if (model != null)
{
IfcBuilding building = CreateBuilding(model, "Default Building");
IfcWallStandardCase wall = CreateWall(model, 4000, 300, 2400);
if (wall != null) AddPropertiesToWall(model, wall);
using (var txn = model.BeginTransaction("Add Wall"))
{
building.AddElement(wall);
txn.Commit();
}
if (wall != null)
{
try
{
Console.WriteLine("Standard Wall successfully created....");
//write the Ifc File
model.SaveAs("HelloWallIfc4.ifc", IfcStorageType.Ifc);
Console.WriteLine("HelloWallIfc4.ifc has been successfully written");
}
catch (Exception e)
{
Console.WriteLine("Failed to save HelloWall.ifc");
Console.WriteLine(e.Message);
}
}
}
else
{
Console.WriteLine("Failed to initialise the model");
}
}
Console.WriteLine("Press any key to exit to view the IFC file....");
Console.ReadKey();
LaunchNotepad("HelloWallIfc4.ifc");
return 0;
}
private static void LaunchNotepad(string fileName)
{
Process p;
try
{
p = new Process {StartInfo = {FileName = fileName, CreateNoWindow = false}};
p.Start();
}
catch (Exception ex)
{
Console.WriteLine("Exception Occurred :{0},{1}",
ex.Message, ex.StackTrace);
}
}
private static IfcBuilding CreateBuilding(IfcStore model, string name)
{
using (var txn = model.BeginTransaction("Create Building"))
{
var building = model.Instances.New<IfcBuilding>();
building.Name = name;
building.CompositionType = IfcElementCompositionEnum.ELEMENT;
var localPlacement = model.Instances.New<IfcLocalPlacement>();
building.ObjectPlacement = localPlacement;
var placement = model.Instances.New<IfcAxis2Placement3D>();
localPlacement.RelativePlacement = placement;
placement.Location = model.Instances.New<IfcCartesianPoint>(p=>p.SetXYZ(0,0,0));
//get the project there should only be one and it should exist
var project = model.Instances.OfType<IfcProject>().FirstOrDefault();
project?.AddBuilding(building);
txn.Commit();
return building;
}
}
/// <summary>
/// Sets up the basic parameters any model must provide, units, ownership etc
/// </summary>
/// <param name="projectName">Name of the project</param>
/// <returns></returns>
private static IfcStore CreateandInitModel(string projectName)
{
//first we need to set up some credentials for ownership of data in the new model
var credentials = new XbimEditorCredentials
{
ApplicationDevelopersName = "xbim developer",
ApplicationFullName = "Hello Wall Application",
ApplicationIdentifier = "HelloWall.exe",
ApplicationVersion = "1.0",
EditorsFamilyName = "Team",
EditorsGivenName = "xbim",
EditorsOrganisationName = "xbim developer"
};
//now we can create an IfcStore, it is in Ifc4 format and will be held in memory rather than in a database
//database is normally better in performance terms if the model is large >50MB of Ifc or if robust transactions are required
var model = IfcStore.Create(credentials, IfcSchemaVersion.Ifc4,XbimStoreType.InMemoryModel);
//Begin a transaction as all changes to a model are ACID
using (var txn = model.BeginTransaction("Initialise Model"))
{
//create a project
var project = model.Instances.New<IfcProject>();
//set the units to SI (mm and metres)
project.Initialize(ProjectUnits.SIUnitsUK);
project.Name = projectName;
//now commit the changes, else they will be rolled back at the end of the scope of the using statement
txn.Commit();
}
return model;
}
/// <summary>
/// This creates a wall and it's geometry, many geometric representations are possible and extruded rectangular footprint is chosen as this is commonly used for standard case walls
/// </summary>
/// <param name="model"></param>
/// <param name="length">Length of the rectangular footprint</param>
/// <param name="width">Width of the rectangular footprint (width of the wall)</param>
/// <param name="height">Height to extrude the wall, extrusion is vertical</param>
/// <returns></returns>
static private IfcWallStandardCase CreateWall(IfcStore model, double length, double width, double height)
{
//
//begin a transaction
using (var txn = model.BeginTransaction("Create Wall"))
{
var wall = model.Instances.New<IfcWallStandardCase>();
wall.Name = "A Standard rectangular wall";
//represent wall as a rectangular profile
var rectProf = model.Instances.New<IfcRectangleProfileDef>();
rectProf.ProfileType = IfcProfileTypeEnum.AREA;
rectProf.XDim = width;
rectProf.YDim = length;
var insertPoint = model.Instances.New<IfcCartesianPoint>();
insertPoint.SetXY(0, 400); //insert at arbitrary position
rectProf.Position = model.Instances.New<IfcAxis2Placement2D>();
rectProf.Position.Location = insertPoint;
//model as a swept area solid
var body = model.Instances.New<IfcExtrudedAreaSolid>();
body.Depth = height;
body.SweptArea = rectProf;
body.ExtrudedDirection = model.Instances.New<IfcDirection>();
body.ExtrudedDirection.SetXYZ(0, 0, 1);
//parameters to insert the geometry in the model
var origin = model.Instances.New<IfcCartesianPoint>();
origin.SetXYZ(0, 0, 0);
body.Position = model.Instances.New<IfcAxis2Placement3D>();
body.Position.Location = origin;
//Create a Definition shape to hold the geometry
var shape = model.Instances.New<IfcShapeRepresentation>();
var modelContext = model.Instances.OfType<IfcGeometricRepresentationContext>().FirstOrDefault();
shape.ContextOfItems = modelContext;
shape.RepresentationType = "SweptSolid";
shape.RepresentationIdentifier = "Body";
shape.Items.Add(body);
//Create a Product Definition and add the model geometry to the wall
var rep = model.Instances.New<IfcProductDefinitionShape>();
rep.Representations.Add(shape);
wall.Representation = rep;
//now place the wall into the model
var lp = model.Instances.New<IfcLocalPlacement>();
var ax3D = model.Instances.New<IfcAxis2Placement3D>();
ax3D.Location = origin;
ax3D.RefDirection = model.Instances.New<IfcDirection>();
ax3D.RefDirection.SetXYZ(0, 1, 0);
ax3D.Axis = model.Instances.New<IfcDirection>();
ax3D.Axis.SetXYZ(0, 0, 1);
lp.RelativePlacement = ax3D;
wall.ObjectPlacement = lp;
// Where Clause: The IfcWallStandard relies on the provision of an IfcMaterialLayerSetUsage
var ifcMaterialLayerSetUsage = model.Instances.New<IfcMaterialLayerSetUsage>();
var ifcMaterialLayerSet = model.Instances.New<IfcMaterialLayerSet>();
var ifcMaterialLayer = model.Instances.New<IfcMaterialLayer>();
ifcMaterialLayer.LayerThickness = 10;
ifcMaterialLayerSet.MaterialLayers.Add(ifcMaterialLayer);
ifcMaterialLayerSetUsage.ForLayerSet = ifcMaterialLayerSet;
ifcMaterialLayerSetUsage.LayerSetDirection = IfcLayerSetDirectionEnum.AXIS2;
ifcMaterialLayerSetUsage.DirectionSense = IfcDirectionSenseEnum.NEGATIVE;
ifcMaterialLayerSetUsage.OffsetFromReferenceLine = 150;
// Add material to wall
var material = model.Instances.New<IfcMaterial>();
material.Name = "some material";
var ifcRelAssociatesMaterial = model.Instances.New<IfcRelAssociatesMaterial>();
ifcRelAssociatesMaterial.RelatingMaterial = material;
ifcRelAssociatesMaterial.RelatedObjects.Add(wall);
ifcRelAssociatesMaterial.RelatingMaterial = ifcMaterialLayerSetUsage;
// IfcPresentationLayerAssignment is required for CAD presentation in IfcWall or IfcWallStandardCase
var ifcPresentationLayerAssignment = model.Instances.New<IfcPresentationLayerAssignment>();
ifcPresentationLayerAssignment.Name = "some ifcPresentationLayerAssignment";
ifcPresentationLayerAssignment.AssignedItems.Add(shape);
// linear segment as IfcPolyline with two points is required for IfcWall
var ifcPolyline = model.Instances.New<IfcPolyline>();
var startPoint = model.Instances.New<IfcCartesianPoint>();
startPoint.SetXY(0, 0);
var endPoint = model.Instances.New<IfcCartesianPoint>();
endPoint.SetXY(4000, 0);
ifcPolyline.Points.Add(startPoint);
ifcPolyline.Points.Add(endPoint);
var shape2D = model.Instances.New<IfcShapeRepresentation>();
shape2D.ContextOfItems = modelContext;
shape2D.RepresentationIdentifier = "Axis";
shape2D.RepresentationType = "Curve2D";
shape2D.Items.Add(ifcPolyline);
rep.Representations.Add(shape2D);
txn.Commit();
return wall;
}
}
/// <summary>
/// Add some properties to the wall,
/// </summary>
/// <param name="model">XbimModel</param>
/// <param name="wall"></param>
static private void AddPropertiesToWall(IfcStore model, IfcWallStandardCase wall)
{
using (var txn = model.BeginTransaction("Create Wall"))
{
CreateElementQuantity(model, wall);
CreateSimpleProperty(model, wall);
txn.Commit();
}
}
private static void CreateSimpleProperty(IfcStore model, IfcWallStandardCase wall)
{
var ifcPropertySingleValue = model.Instances.New<IfcPropertySingleValue>(psv =>
{
psv.Name = "IfcPropertySingleValue:Time";
psv.Description = "";
psv.NominalValue = new IfcTimeMeasure(150.0);
psv.Unit = model.Instances.New<IfcSIUnit>(siu =>
{
siu.UnitType = IfcUnitEnum.TIMEUNIT;
siu.Name = IfcSIUnitName.SECOND;
});
});
var ifcPropertyEnumeratedValue = model.Instances.New<IfcPropertyEnumeratedValue>(pev =>
{
pev.Name = "IfcPropertyEnumeratedValue:Music";
pev.EnumerationReference = model.Instances.New<IfcPropertyEnumeration>(pe =>
{
pe.Name = "Notes";
pe.EnumerationValues.Add(new IfcLabel("Do"));
pe.EnumerationValues.Add(new IfcLabel("Re"));
pe.EnumerationValues.Add(new IfcLabel("Mi"));
pe.EnumerationValues.Add(new IfcLabel("Fa"));
pe.EnumerationValues.Add(new IfcLabel("So"));
pe.EnumerationValues.Add(new IfcLabel("La"));
pe.EnumerationValues.Add(new IfcLabel("Ti"));
});
pev.EnumerationValues.Add(new IfcLabel("Do"));
pev.EnumerationValues.Add(new IfcLabel("Re"));
pev.EnumerationValues.Add(new IfcLabel("Mi"));
});
var ifcPropertyBoundedValue = model.Instances.New<IfcPropertyBoundedValue>(pbv =>
{
pbv.Name = "IfcPropertyBoundedValue:Mass";
pbv.Description = "";
pbv.UpperBoundValue = new IfcMassMeasure(5000.0);
pbv.LowerBoundValue = new IfcMassMeasure(1000.0);
pbv.Unit = model.Instances.New<IfcSIUnit>(siu =>
{
siu.UnitType = IfcUnitEnum.MASSUNIT;
siu.Name = IfcSIUnitName.GRAM;
siu.Prefix = IfcSIPrefix.KILO;
});
});
var definingValues = new List<IfcReal> { new IfcReal(100.0), new IfcReal(200.0), new IfcReal(400.0), new IfcReal(800.0), new IfcReal(1600.0), new IfcReal(3200.0), };
var definedValues = new List<IfcReal> { new IfcReal(20.0), new IfcReal(42.0), new IfcReal(46.0), new IfcReal(56.0), new IfcReal(60.0), new IfcReal(65.0), };
var ifcPropertyTableValue = model.Instances.New<IfcPropertyTableValue>(ptv =>
{
ptv.Name = "IfcPropertyTableValue:Sound";
foreach (var item in definingValues)
{
ptv.DefiningValues.Add(item);
}
foreach (var item in definedValues)
{
ptv.DefinedValues.Add(item);
}
ptv.DefinedUnit = model.Instances.New<IfcContextDependentUnit>(cd =>
{
cd.Dimensions = model.Instances.New<IfcDimensionalExponents>(de =>
{
de.LengthExponent = 0;
de.MassExponent = 0;
de.TimeExponent = 0;
de.ElectricCurrentExponent = 0;
de.ThermodynamicTemperatureExponent = 0;
de.AmountOfSubstanceExponent = 0;
de.LuminousIntensityExponent = 0;
});
cd.UnitType = IfcUnitEnum.FREQUENCYUNIT;
cd.Name = "dB";
});
});
var listValues = new List<IfcLabel> { new IfcLabel("Red"), new IfcLabel("Green"), new IfcLabel("Blue"), new IfcLabel("Pink"), new IfcLabel("White"), new IfcLabel("Black"), };
var ifcPropertyListValue = model.Instances.New<IfcPropertyListValue>(plv =>
{
plv.Name = "IfcPropertyListValue:Colours";
foreach (var item in listValues)
{
plv.ListValues.Add(item);
}
});
var ifcMaterial = model.Instances.New<IfcMaterial>(m =>
{
m.Name = "Brick";
});
var ifcPrValueMaterial = model.Instances.New<IfcPropertyReferenceValue>(prv =>
{
prv.Name = "IfcPropertyReferenceValue:Material";
prv.PropertyReference = ifcMaterial;
});
var ifcMaterialList = model.Instances.New<IfcMaterialList>(ml =>
{
ml.Materials.Add(ifcMaterial);
ml.Materials.Add(model.Instances.New<IfcMaterial>(m =>{m.Name = "Cavity";}));
ml.Materials.Add(model.Instances.New<IfcMaterial>(m => { m.Name = "Block"; }));
});
var ifcMaterialLayer = model.Instances.New<IfcMaterialLayer>(ml =>
{
ml.Material = ifcMaterial;
ml.LayerThickness = 100.0;
});
var ifcPrValueMatLayer = model.Instances.New<IfcPropertyReferenceValue>(prv =>
{
prv.Name = "IfcPropertyReferenceValue:MaterialLayer";
prv.PropertyReference = ifcMaterialLayer;
});
var ifcDocumentReference = model.Instances.New<IfcDocumentReference>(dr =>
{
dr.Name = "Document";
dr.Location = "c://Documents//TheDoc.Txt";
});
var ifcPrValueRef = model.Instances.New<IfcPropertyReferenceValue>(prv =>
{
prv.Name = "IfcPropertyReferenceValue:Document";
prv.PropertyReference = ifcDocumentReference;
});
var ifcTimeSeries = model.Instances.New<IfcRegularTimeSeries>(ts =>
{
ts.Name = "Regular Time Series";
ts.Description = "Time series of events";
ts.StartTime = new IfcDateTime("2015-02-14T12:01:01");
ts.EndTime = new IfcDateTime("2015-05-15T12:01:01");
ts.TimeSeriesDataType = IfcTimeSeriesDataTypeEnum.CONTINUOUS;
ts.DataOrigin = IfcDataOriginEnum.MEASURED;
ts.TimeStep = 604800; //7 days in secs
});
var ifcPrValueTimeSeries = model.Instances.New<IfcPropertyReferenceValue>(prv =>
{
prv.Name = "IfcPropertyReferenceValue:TimeSeries";
prv.PropertyReference = ifcTimeSeries;
});
var ifcAddress = model.Instances.New<IfcPostalAddress>(a =>
{
a.InternalLocation = "Room 101";
a.AddressLines.AddRange(new[] { new IfcLabel("12 New road"), new IfcLabel("DoxField" ) });
a.Town = "Sunderland";
a.PostalCode = "DL01 6SX";
});
var ifcPrValueAddress = model.Instances.New<IfcPropertyReferenceValue>(prv =>
{
prv.Name = "IfcPropertyReferenceValue:Address";
prv.PropertyReference = ifcAddress;
});
var ifcTelecomAddress = model.Instances.New<IfcTelecomAddress>(a =>
{
a.TelephoneNumbers.Add(new IfcLabel("01325 6589965"));
a.ElectronicMailAddresses.Add(new IfcLabel("bob#bobsworks.com"));
});
var ifcPrValueTelecom = model.Instances.New<IfcPropertyReferenceValue>(prv =>
{
prv.Name = "IfcPropertyReferenceValue:Telecom";
prv.PropertyReference = ifcTelecomAddress;
});
//lets create the IfcElementQuantity
var ifcPropertySet = model.Instances.New<IfcPropertySet>(ps =>
{
ps.Name = "Test:IfcPropertySet";
ps.Description = "Property Set";
ps.HasProperties.Add(ifcPropertySingleValue);
ps.HasProperties.Add(ifcPropertyEnumeratedValue);
ps.HasProperties.Add(ifcPropertyBoundedValue);
ps.HasProperties.Add(ifcPropertyTableValue);
ps.HasProperties.Add(ifcPropertyListValue);
ps.HasProperties.Add(ifcPrValueMaterial);
ps.HasProperties.Add(ifcPrValueMatLayer);
ps.HasProperties.Add(ifcPrValueRef);
ps.HasProperties.Add(ifcPrValueTimeSeries);
ps.HasProperties.Add(ifcPrValueAddress);
ps.HasProperties.Add(ifcPrValueTelecom);
});
//need to create the relationship
model.Instances.New<IfcRelDefinesByProperties>(rdbp =>
{
rdbp.Name = "Property Association";
rdbp.Description = "IfcPropertySet associated to wall";
rdbp.RelatedObjects.Add(wall);
rdbp.RelatingPropertyDefinition = ifcPropertySet;
});
}
private static void CreateElementQuantity(IfcStore model, IfcWallStandardCase wall)
{
//Create a IfcElementQuantity
//first we need a IfcPhysicalSimpleQuantity,first will use IfcQuantityLength
var ifcQuantityArea = model.Instances.New<IfcQuantityLength>(qa =>
{
qa.Name = "IfcQuantityArea:Area";
qa.Description = "";
qa.Unit = model.Instances.New<IfcSIUnit>(siu =>
{
siu.UnitType = IfcUnitEnum.LENGTHUNIT;
siu.Prefix = IfcSIPrefix.MILLI;
siu.Name = IfcSIUnitName.METRE;
});
qa.LengthValue = 100.0;
});
//next quantity IfcQuantityCount using IfcContextDependentUnit
var ifcContextDependentUnit = model.Instances.New<IfcContextDependentUnit>(cd =>
{
cd.Dimensions = model.Instances.New<IfcDimensionalExponents>(de =>
{
de.LengthExponent = 1;
de.MassExponent = 0;
de.TimeExponent = 0;
de.ElectricCurrentExponent = 0;
de.ThermodynamicTemperatureExponent = 0;
de.AmountOfSubstanceExponent = 0;
de.LuminousIntensityExponent = 0;
});
cd.UnitType = IfcUnitEnum.LENGTHUNIT;
cd.Name = "Elephants";
});
var ifcQuantityCount = model.Instances.New<IfcQuantityCount>(qc =>
{
qc.Name = "IfcQuantityCount:Elephant";
qc.CountValue = 12;
qc.Unit = ifcContextDependentUnit;
});
//next quantity IfcQuantityLength using IfcConversionBasedUnit
var ifcConversionBasedUnit = model.Instances.New<IfcConversionBasedUnit>(cbu =>
{
cbu.ConversionFactor = model.Instances.New<IfcMeasureWithUnit>(mu =>
{
mu.ValueComponent = new IfcRatioMeasure(25.4);
mu.UnitComponent = model.Instances.New<IfcSIUnit>(siu =>
{
siu.UnitType = IfcUnitEnum.LENGTHUNIT;
siu.Prefix = IfcSIPrefix.MILLI;
siu.Name = IfcSIUnitName.METRE;
});
});
cbu.Dimensions = model.Instances.New<IfcDimensionalExponents>(de =>
{
de.LengthExponent = 1;
de.MassExponent = 0;
de.TimeExponent = 0;
de.ElectricCurrentExponent = 0;
de.ThermodynamicTemperatureExponent = 0;
de.AmountOfSubstanceExponent = 0;
de.LuminousIntensityExponent = 0;
});
cbu.UnitType = IfcUnitEnum.LENGTHUNIT;
cbu.Name = "Inch";
});
var ifcQuantityLength = model.Instances.New<IfcQuantityLength>(qa =>
{
qa.Name = "IfcQuantityLength:Length";
qa.Description = "";
qa.Unit = ifcConversionBasedUnit;
qa.LengthValue = 24.0;
});
//lets create the IfcElementQuantity
var ifcElementQuantity = model.Instances.New<IfcElementQuantity>(eq =>
{
eq.Name = "Test:IfcElementQuantity";
eq.Description = "Measurement quantity";
eq.Quantities.Add(ifcQuantityArea);
eq.Quantities.Add(ifcQuantityCount);
eq.Quantities.Add(ifcQuantityLength);
});
//need to create the relationship
model.Instances.New<IfcRelDefinesByProperties>(rdbp =>
{
rdbp.Name = "Area Association";
rdbp.Description = "IfcElementQuantity associated to wall";
rdbp.RelatedObjects.Add(wall);
rdbp.RelatingPropertyDefinition = ifcElementQuantity;
});
}
}
}
I will also have questions about assigning materials in the future but for now, I really need help with the opening.
I generate lines of rows to insert:
private StringReader MapWithHeaders(StreamReader reader, int clientId, HeadersMap dataHeadersMap, string delimiter, int uploadDataId, string dateFormat)
{
var lines = new HashSet<string?>();
var headers = new List<string>();
var i = 0;
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var csvLineInOrder = "";
var values = line.Split(delimiter).Select(x => x.Trim(new[] { '/', '"' })).ToList();
if (i == 0)
{
headers = values.Select(x => x.Trim(new[] { '/', '"' })).ToList();
csvLineInOrder = String.Join(delimiter, DataHeaders);
i++;
}
else
{
var campaignColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.CampaignColumnName));
var clusterColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.ClusterColumnName));
var userIdColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.UserIdColumnName));
var channelColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.ChannelColumnName));
csvLineInOrder = values[campaignColumnNameId] + delimiter + values[clusterColumnNameId] + delimiter +
values[userIdColumnNameId] + delimiter + values[channelColumnNameId];
/* e.g.: ,,11111,,7,,674 */
}
lines.Add(csvLineInOrder);
}
var stringCsv = string.Join("\n", lines);
return new StringReader(stringCsv);
}
then I generate csvDataReader:
private async Task BulkImportCsvExtra(Stream file, int clientId, HeadersMap dataExtraHeadersMap, string delimiter, int uploadDataId, string dateFormat)
{
using (var reader = new StreamReader(file))
{
var config = new CsvConfiguration(CultureInfo.InvariantCulture);
config.BadDataFound = null;
config.Delimiter = delimiter;
// doesn't get triggered?
config.TypeConverterCache.AddConverter<string>(new EmptyAsNullConverter());
var csvStreamReader = MapWithHeaders(reader, clientId, dataExtraHeadersMap, delimiter, uploadDataId, dateFormat);
using (var csv = new CsvReader(csvStreamReader, config))
{
var dataReader = new CsvDataReader(csv);
// Need for header parsing
csv.ReadHeader();
if (!HeadersValid(csv.Context.HeaderRecord, DataHeadersExtra))
throw new Exception(ExceptionCode.Import.InvalidHeaders);
await _repository.BulkAdd(dataReader);
}
}
}
and finally use bulkCopy to add data:
public async Task BulkAdd(IDataReader data)
{
if (Connection.State == ConnectionState.Broken || Connection.State == ConnectionState.Closed)
{
await Connection.OpenAsync();
}
using (SqlBulkCopy bulk = new SqlBulkCopy(Connection))
{
bulk.DestinationTableName = GetTableName();
bulk.BatchSize = BATCH_SIZE;
bulk.BulkCopyTimeout = 0; // for infinity write 0
bulk.EnableStreaming = true;
await bulk.WriteToServerAsync(data);
}
}
however empty values are written as empty strings rather than NULL even though I have default null constraints on database side. I've been stuck here for ages. any Ideas how to fix?
I tried adding null converter:
public class EmptyAsNullConverter : CsvHelper.TypeConversion.StringConverter
{
public override object ConvertFromString(string text, IReaderRow row, MemberMapData memberMapData)
{
if (string.IsNullOrWhiteSpace(text))
return null;
return text;
}
}
but it's not used even thouhg I add it as type converter and I don't know how to trigger it
I need to refactor the below code so that the deleted_at logic will be outside of the foreach (var app in data) loop. I tried to create the list guids and then add guids to it but its not working because model.resources is inside the loop and it is still deleting all the apps.
I need deleted_at logic outside because I'm trying to delete all apps which are in the database but are not in new data that I'm receiving from API.
If you have a better approach on my code I would love to see that, Thank you.
public async Task GetBuilds()
{
var data = new List<GetBuildTempClass>();
var guids = new List<GetBuildTempClass>();
using (var scope = _scopeFactory.CreateScope())
{
var _DBcontext = scope.ServiceProvider.GetRequiredService<PCFStatusContexts>();
foreach (var app in data)
{
var request = new HttpRequestMessage(HttpMethod.Get,
"apps/" + app.AppGuid + "/builds?per_page=200&order_by=updated_at");
var response = await _client_SB.SendAsync(request);
var json = await response.Content.ReadAsStringAsync();
BuildsClass.BuildsRootObject model =
JsonConvert.DeserializeObject<BuildsClass.BuildsRootObject>(json);
foreach (var item in model.resources)
{
var x = _DBcontext.Builds.FirstOrDefault(o =>
o.Guid == Guid.Parse(item.guid));
if (x == null)
{
_DBcontext.Builds.Add(new Builds
{
Guid = Guid.Parse(item.guid),
State = item.state,
CreatedAt = item.created_at,
UpdatedAt = item.updated_at,
Error = item.error,
CreatedByGuid = Guid.Parse(item.created_by.guid),
CreatedByName = item.created_by.name,
CreatedByEmail = item.created_by.email,
AppGuid = app.AppGuid,
AppName = app.AppName,
Foundation = 2,
Timestamp = DateTime.Now
});
}
else if (x.UpdatedAt != item.updated_at)
{
x.State = item.state;
x.UpdatedAt = item.updated_at;
x.Timestamp = DateTime.Now;
}
var sqlresult = new GetBuildTempClass
{
AppGuid = Guid.Parse(item.guid)
};
guids.Add(sqlresult);
}
//var guids = model.resources.Select(r => Guid.Parse(r.guid));
var builds = _DBcontext.Builds.Where(o =>
guids.Contains(o.Guid) == false &&
o.Foundation == 2 && o.DeletedAt == null);
foreach (var build_item in builds)
{
build_item.DeletedAt = DateTime.Now;
}
}
await _DBcontext.SaveChangesAsync();
}
}
I got it working I added var guids = new List < Guid > (); list to store data,
added guids.Add(Guid.Parse(item.guid)); to populate the list and finally wrote var builds = _DBcontext.Builds.Where(o = >guids.Contains(o.Guid) == false && o.Foundation == 2 && o.DeletedAt == null); outside the loop.
If anyone has a better suggestion please add a new answer.
public async Task GetBuilds() {
var data = new List < GetBuildTempClass > ();
var guids = new List < Guid > ();
using(var scope = _scopeFactory.CreateScope()) {
var _DBcontext = scope.ServiceProvider.GetRequiredService < PCFStatusContexts > ();
foreach(var app in data) {
var request = new HttpRequestMessage(HttpMethod.Get, "apps/" + app.AppGuid + "/builds?per_page=200&order_by=updated_at");
var response = await _client_SB.SendAsync(request);
var json = await response.Content.ReadAsStringAsync();
BuildsClass.BuildsRootObject model = JsonConvert.DeserializeObject < BuildsClass.BuildsRootObject > (json);
foreach(var item in model.resources) {
var x = _DBcontext.Builds.FirstOrDefault(o = >o.Guid == Guid.Parse(item.guid));
if (x == null) {
_DBcontext.Builds.Add(new Builds {
Guid = Guid.Parse(item.guid),
State = item.state,
CreatedAt = item.created_at,
UpdatedAt = item.updated_at,
Error = item.error,
CreatedByGuid = Guid.Parse(item.created_by.guid),
CreatedByName = item.created_by.name,
CreatedByEmail = item.created_by.email,
AppGuid = app.AppGuid,
AppName = app.AppName,
Foundation = 2,
Timestamp = DateTime.Now
});
}
else if (x.UpdatedAt != item.updated_at) {
x.State = item.state;
x.UpdatedAt = item.updated_at;
x.Timestamp = DateTime.Now;
}
guids.Add(Guid.Parse(item.guid));
}
}
var builds = _DBcontext.Builds.Where(o = >guids.Contains(o.Guid) == false && o.Foundation == 2 && o.DeletedAt == null);
foreach(var build_item in builds) {
build_item.DeletedAt = DateTime.Now;
}
await _DBcontext.SaveChangesAsync();
}
}