I have two different function to handle two different types of my input text file. One text file with double quotes and one without double quotes.
I wanted to know how can i combine these two functions to a common single function where i can handle in a more efficient way
Code:
//this the function to handle text file without double quotes
public void stack1()
{
string old;
string iniPath = Application.StartupPath + "\\list.ini";
bool isDeleteSectionFound = false;
List<string> deleteCodeList = new List<string>();
using (StreamReader sr = File.OpenText(iniPath))
{
while ((old = sr.ReadLine()) != null)
{
if (old.Trim().Equals("[DELETE]"))
{
isDeleteSectionFound = true;
}
if (isDeleteSectionFound && !old.Trim().Equals("[DELETE]"))
{
deleteCodeList.Add(old.Trim());
}
}
}
StringBuilder sb = new StringBuilder();
using (StreamReader reader = File.OpenText(textBox1.Text))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var value = line.Split('\t');
bool deleteLine = value.Any(v => deleteCodeList.Any(w => v.Equals(w)));
if (!deleteLine)
{
sb.Append(line + Environment.NewLine);
}
}
}
File.WriteAllText(textBox1.Text, sb.ToString());
//return;
}
//this the function to handle text file with double quotes
public void stack()
{
string old;
string iniPath = Application.StartupPath + "\\list.ini";
bool isDeleteSectionFound = false;
List<string> deleteCodeList = new List<string>();
using (StreamReader sr = File.OpenText(iniPath))
{
while ((old = sr.ReadLine()) != null)
{
if (old.Trim().Equals("[DELETE]"))
{
isDeleteSectionFound = true;
}
if (isDeleteSectionFound && !old.Trim().Equals("[DELETE]"))
{
deleteCodeList.Add(old.Trim());
}
}
}
StringBuilder sb = new StringBuilder();
using (StreamReader reader = File.OpenText(textBox1.Text))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split('\t').Select(v => v.Trim(' ', '"'));
bool deleteLines = values.Any(v => deleteCodeList.Any(w => v.Equals(w)));
if (!deleteLines)
{
sb.Append(line + Environment.NewLine);
}
}
}
File.WriteAllText(textBox1.Text, sb.ToString());
MessageBox.Show("finish");
}
The only difference between these two functions is this line:
// stack1 function
var value = line.Split('\t');
// stack2 function
var values = line.Split('\t').Select(v => v.Trim(' ', '"'));
The simplest way would probably be to add a parameter to your method, and then add the check after the split:
public void Split(bool shouldTrimQuotes)
{
...
IEnumerable<string> value = line.Split('\t');
if (shouldTrimQuotes)
{
value = value.Select(v => v.Trim(' ', '"'));
}
...
}
In one case, you would pass true as the parameter (which will cause quotes to be trimmed), while in the second one you would pass false to indicate you don't want to trim them:
// split, but don't trim quotes before comparison
Split(shouldTrimQuotes: false);
// split, trim quotes before comparison
Split(shouldTrimQuotes: true);
You might also play a bit and try to refactor the whole thing, trying to extract smaller general pieces of code into separate methods which might make it clearer what they are doing. This is one approach, for example:
// rewrites the specified file, removing all lines matched by the predicate
public static void RemoveLinesFromFile(string filename, Func<string, bool> match)
{
var linesToKeep = File.ReadAllLines(filename)
.Where(line => match(line))
.ToList();
File.WriteAllLines(filename, linesToKeep);
}
// gets the list of "delete codes" from the specified ini file
public IList<string> GetDeleteCodeList(string iniPath)
{
return File.ReadLines(iniPath)
.SkipWhile(l => l.Trim() != "[DELETE]")
.Skip(1).ToList();
}
// removes lines from a tab-delimited file, where the specified listOfCodes contains
// at least one of the tokens inside that line
public static void RemoveLinesUsingCodeList(
string filename,
IList<string> listOfCodes,
bool shouldTrimQuotes)
{
RemoveLinesFromFile(filename, line =>
{
IEnumerable<string> tokens = line.Split('\t');
if (shouldTrimQuotes)
{
tokens = tokens.Select(v => v.Trim(' ', '"'));
}
return (tokens.Any(t => listOfCodes.Any(t.Equals)));
});
}
Related
I generate lines of rows to insert:
private StringReader MapWithHeaders(StreamReader reader, int clientId, HeadersMap dataHeadersMap, string delimiter, int uploadDataId, string dateFormat)
{
var lines = new HashSet<string?>();
var headers = new List<string>();
var i = 0;
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var csvLineInOrder = "";
var values = line.Split(delimiter).Select(x => x.Trim(new[] { '/', '"' })).ToList();
if (i == 0)
{
headers = values.Select(x => x.Trim(new[] { '/', '"' })).ToList();
csvLineInOrder = String.Join(delimiter, DataHeaders);
i++;
}
else
{
var campaignColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.CampaignColumnName));
var clusterColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.ClusterColumnName));
var userIdColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.UserIdColumnName));
var channelColumnNameId = headers.FindIndex(x => x.Equals(dataHeadersMap.ChannelColumnName));
csvLineInOrder = values[campaignColumnNameId] + delimiter + values[clusterColumnNameId] + delimiter +
values[userIdColumnNameId] + delimiter + values[channelColumnNameId];
/* e.g.: ,,11111,,7,,674 */
}
lines.Add(csvLineInOrder);
}
var stringCsv = string.Join("\n", lines);
return new StringReader(stringCsv);
}
then I generate csvDataReader:
private async Task BulkImportCsvExtra(Stream file, int clientId, HeadersMap dataExtraHeadersMap, string delimiter, int uploadDataId, string dateFormat)
{
using (var reader = new StreamReader(file))
{
var config = new CsvConfiguration(CultureInfo.InvariantCulture);
config.BadDataFound = null;
config.Delimiter = delimiter;
// doesn't get triggered?
config.TypeConverterCache.AddConverter<string>(new EmptyAsNullConverter());
var csvStreamReader = MapWithHeaders(reader, clientId, dataExtraHeadersMap, delimiter, uploadDataId, dateFormat);
using (var csv = new CsvReader(csvStreamReader, config))
{
var dataReader = new CsvDataReader(csv);
// Need for header parsing
csv.ReadHeader();
if (!HeadersValid(csv.Context.HeaderRecord, DataHeadersExtra))
throw new Exception(ExceptionCode.Import.InvalidHeaders);
await _repository.BulkAdd(dataReader);
}
}
}
and finally use bulkCopy to add data:
public async Task BulkAdd(IDataReader data)
{
if (Connection.State == ConnectionState.Broken || Connection.State == ConnectionState.Closed)
{
await Connection.OpenAsync();
}
using (SqlBulkCopy bulk = new SqlBulkCopy(Connection))
{
bulk.DestinationTableName = GetTableName();
bulk.BatchSize = BATCH_SIZE;
bulk.BulkCopyTimeout = 0; // for infinity write 0
bulk.EnableStreaming = true;
await bulk.WriteToServerAsync(data);
}
}
however empty values are written as empty strings rather than NULL even though I have default null constraints on database side. I've been stuck here for ages. any Ideas how to fix?
I tried adding null converter:
public class EmptyAsNullConverter : CsvHelper.TypeConversion.StringConverter
{
public override object ConvertFromString(string text, IReaderRow row, MemberMapData memberMapData)
{
if (string.IsNullOrWhiteSpace(text))
return null;
return text;
}
}
but it's not used even thouhg I add it as type converter and I don't know how to trigger it
New to C#, and having trouble finding ways to compare data so far collected from conf file, and outputting it to either text or CSV.
I so far have the skeleton of data extraction code from said conf file, however as I'm new to C# and coding overall, I'm having trouble understanding how to reference that data or compare it.
So far have tried File.WriteAllLiness and defining a variable, but not sure which element to parse, or at which point in the code I should introduce it.
Nothing to hide really, so here's the full output so far:
namespace CompareVal
{
class Program
{
static void Main(string[] args)
{
var lines = File.ReadAllLines(#"D:\*\*\Cleanup\Script Project\Test-Raw-Conf.txt");
var ipAddresses = GetIPAddresses(lines);
var routes = GetRoutes(lines);
var ipRules = GetIPRules(lines);
Console.WriteLine ();
}
static Dictionary<string, string[]> GetIPAddresses(string[] lines)
{
var result = new Dictionary<string, string[]>();
foreach (var line in lines)
{
if (!line.StartsWith("add IPAddress"))
{
continue;
}
Match match;
if (line.Contains("Address=\""))
{
match = Regex.Match(line, "add IPAddress (.*?) Address=\"(.*?)\"");
}
else
{
match = Regex.Match(line, "add IPAddress (.*?) Address=(.*?)$");
}
var name = match.Groups[1].Value;
var value = match.Groups[2].Value;
var items = value.Replace(" ", "").Split(',');
result.Add(name, items);
}
return result;
}
static List<Route> GetRoutes(string[] lines)
{
var result = new List<Route>();
string currentRoutingTable = null;
foreach (var line in lines)
{
if (line.StartsWith("cc RoutingTable"))
{
currentRoutingTable = line.Split(' ')[2].Trim();
}
if (line == "cc .." && currentRoutingTable != null)
{
currentRoutingTable = null;
}
if (line.StartsWith(" add Route"))
{
var #interface = Regex.Match(line, "Interface=(.*?) ").Groups[1].Value;
var gateway = Regex.Match(line, "Gateway=(.*?) ").Groups[1].Value;
var network = Regex.Match(line, "Network=(.*?) ").Groups[1].Value;
result.Add(new Route
{
RoutingTable = currentRoutingTable,
Interface = #interface,
Gateway = gateway,
Network = network
});
}
}
return result;
}
static List<IPRule> GetIPRules(string[] lines)
{
var result = new List<IPRule>();
string currentIPRuleSet = null;
foreach (var line in lines)
{
if (line.StartsWith("cc IPRuleSet"))
{
currentIPRuleSet = line.Split(' ')[2].Trim();
}
if (line == "cc .." && currentIPRuleSet != null)
{
currentIPRuleSet = null;
}
if (line.StartsWith(" add IPRule"))
{
var rule = new IPRule
{
IPRuleSet = currentIPRuleSet,
SourceInterface = GetProperty(line, "SourceInterface"),
DestinationInterface = GetProperty(line, "DestinationInterface"),
};
if (line.Contains("SourceNetwork=\""))
{
rule.SourceNetwork = GetQuotedProperty(line, "SourceNetwork").Replace(" ", "").Split(',');
}
else
{
rule.SourceNetwork = GetProperty(line, "SourceNetwork").Replace(" ", "").Split(',');
}
if (line.Contains("DestinationNetwork=\""))
{
rule.DestinationNetwork = GetQuotedProperty(line, "DestinationNetwork").Replace(" ", "").Split(',');
}
else
{
rule.DestinationNetwork = GetProperty(line, "DestinationNetwork").Replace(" ", "").Split(',');
}
result.Add(rule);
}
}
return result;
}
static string GetProperty(string input, string propertyName)
{
return Regex.Match(input, string.Format("{0}=(.*?) ", propertyName)).Groups[1].Value;
}
static string GetQuotedProperty(string input, string propertyName)
{
return Regex.Match(input, string.Format("{0}=\"(.*?)\" ", propertyName)).Groups[1].Value;
}
class Route
{
public string RoutingTable;
public string Interface;
public string Gateway;
public string Network;
}
class IPRule
{
public string IPRuleSet;
public string SourceInterface;
public string DestinationInterface;
public string[] SourceNetwork;
public string[] DestinationNetwork;
}
}
}
I'm hoping to compare values gathered by IPRule, Route and IPAddress classes, and have a method of outputting each associated value in a list. Each IPAddress is contains a unique string name, but can use any numerical IP address. The idea is to determine when the same IP has been used multiple times, regardless of IPAddress string name, and then compare this to routes, and flag when they are used in IPRules.
For reference, here are some samples of source data:
For IPAddresses, they can be formed in 1 of 2 ways - as a direct IP definition, or as a reference to another IPAddress object (or multi-reference):
add IPAddress Test Address=192.168.1.0/24
IPAddress referencing multiple other IPAddresses:
add IPAddress TestGroup Address="Test1, Test2, Test3"
For routes:
add Route Interface=if5 Gateway=if5_gw Network=Test ProxyARPInterfaces=""
And for IPRules:
add IPRule SourceInterface=if5 DestinationInterface=if3 SourceNetwork=Test1 DestinationNetwork=Test2 Service=dns-all Action=Allow
The above definitions will always follow the same pattern, so the data extraction code has been constructed to expect prefixes to each element, and sort them into their own dictionary or list.
I have text file which contains line like this:
#relation SMILEfeatures
#attribute pcm_LOGenergy_sma_range numeric
#attribute pcm_LOGenergy_sma_maxPos numeric
#attribute pcm_LOGenergy_sma_minPos numeric...
Where are about 6000 lines of these attributes, after attributes where are lines like this:
#data
1.283827e+01,3.800000e+01,2.000000e+00,5.331364e+00
1.850000e+02,4.054457e+01,4.500000e+01,3.200000e+01...
I need to seperate these strings in two different arrays. So far I only managed to store everything in one array.
Here is my code for storing in array:
using (var stream = new FileStream(filePath, FileMode.OpenOrCreate))
{
using (var sr = new StreamReader(stream))
{
String line;
while ((line = sr.ReadLine()) != null)
{
sb.AppendLine(line);
}
}
string allines = sb.ToString();
Console.WriteLine(sb);
}
All strings after #relation SMILEfeatures and contains #attribute are stored in first array. All the strings after #data should are stored in the second array. Hope this is what you wanted.
var relationLineNumbers = new List<int>();
var dataLineNumbers = new List<int>();
var relation = new StringBuilder();
var data = new List<string>();
using (var stream = new FileStream(filepath, FileMode.OpenOrCreate))
{
using (var sr = new StreamReader(stream))
{
string line;
bool isRelation = false;
bool isData = false;
int lineNumber = 0;
while ((line = sr.ReadLine()) != null)
{
lineNumber++;
if (line.StartsWith("#relation SMILEfeatures"))
{
isRelation = true;
isData = false;
continue;
}
if (line.StartsWith("#data"))
{
isData = true;
isRelation = false;
continue;
}
if (isRelation)
{
if (line.StartsWith("#attribute"))
{
relation.Append(line);
relationLineNumbers.Add(lineNumber);
}
}
if (isData)
{
data.AddRange(line.Split(','));
dataLineNumbers.Add(lineNumber);
}
}
}
Console.WriteLine("Relation");
Console.WriteLine(relation.ToString());
Console.WriteLine("Data");
data.ForEach(Console.WriteLine);
All strings which starts with #relation SMILEfeatures and contains #attribute should be stored in first array. Numbers which starts with #data should be stored in second array.
Use string.Contains() and string.StatsWith() for checking.
Read every line and decide in wich array / list you want to put this line
void ReadAndSortInArrays(string fileLocation)
{
List<string> noData = new List<string>();
List<string> Data = new List<string>();
using(StreamReader sr = new StreamReader(fileLocation))
{
string line;
while(!sr.EndOfStream)
{
line = sr.ReadLine();
if(line.StartsWith("#relation") && line.Contains("#attribute"))
{
noData.Add(line);
}
else if(line.StartsWith("#data")
{
Data.Add(line);
}
else
{
// This is stange
}
}
}
var noDataArray = noData.ToArray();
var DataArray = Data.ToArray();
}
But i think that not every line is beginning with "#data"
So you may want to Read all lines and do somethink like this:
string allLines;
using(StreamReader sr = new StreamReader(yourfile))
{
allLines = = sr.ReadToEnd();
}
var arrays = allLines.Split("#data");
// arrays[0] is the part before #data
// arrays[1] is the part after #data (the numbers)
// But array[1] does not contain #data
The question is not really very clear. But my take is, collect all lines that start with #relation or #attribute in one bucket, then collect all number lines in another bucket. I have chosen to ignore the #data lines, as they do not seem to contain any extra information.
Error checking may be performed by making sure that the data lines (i.e. number lines) contain comma separated lists of parsable numerical values.
var dataLines = new List<string>();
var relAttLines = new List<string>();
foreach (var line in File.ReadAllLines())
{
if (line.StartsWith("#relation") || line.StartsWith("#attribute"))
relAttLines.Add(line);
else if (line.StartsWith("#data"))
//ignore these
continue;
else
dataLines.Add(line);
}
I'd like to make below code cleaner (in the eye of the beholder).
var lines = new StringReader(lotsOfIncomingLinesWithNewLineCharacters);
var resultingLines = new List<string>();
string line;
while( (line = lines.ReadLine() ) != null )
{
if( line.Substring(0,5) == "value" )
{
resultingLines.Add(line);
}
}
to something like
var resultingLinesQuery =
lotsOfIncomingLinesWithNewLineCharacters
.Where(s=>s.Substring(0,5) == "value );
Hopefully I have illustrated that I'd prefer to not have the result as a list (to not fill up memory) and that StringReader is not mandatory.
There is the naïve solution to create an extension and move the ReadLine there but I have a feeling there might be a better way.
Basically you need a way of extracting lines from a TextReader. Here's a simple solution which will only iterate once:
public static IEnumerable<string> ReadLines(this TextReader reader)
{
string line;
while ((line = reader.ReadLine()) != null)
{
yield return line;
}
}
You could use that with:
var resultingLinesQuery =
new StringReader(lotsOfIncomingLinesWithNewLineCharacters)
.ReadLines()
.Where(s => s.Substring(0,5) == "value");
But ideally, you should be able to iterate over an IEnumerable<T> more than once. If you only need this for strings, you could use:
public static IEnumerable<string> SplitIntoLines(this string text)
{
using (var reader = new StringReader(text))
{
string line;
while ((line = reader.ReadLine()) != null)
{
yield return line;
}
}
}
Then:
var resultingLinesQuery =
lotsOfIncomingLinesWithNewLineCharacters
.SplitIntoLines()
.Where(s => s.Substring(0,5) == "value");
I have a log file that I am reading into different objects. One object starts at a Line that contains the words "Announce message" and the following lines contain the data that belongs to that message. This entry stops at a line that contains the word "Disposed".
I want to read all the data from between these 2 lines that, contains certain words.
Im currently using a Dictionary because the line with "Announce message" also contains a UID but the following lines contain the data for that UID.
How would you do that?
This is what i have come up with so far.
public static void P2PLogParser(List<FileInfo> fileList)
{
foreach (FileInfo fi in fileList)
{
//Læser alle linier i csv fil
foreach (var line in File.ReadAllLines(fi.FullName))
{
string MeterUID = GetMeterUID(line);
string MimHashcode = GetMimHashcode(line);
string FirmwareUploadStatus = GetFirmwareUploadStatus(line);
string IsKnown = GetIsKnown(line);
DateTime P2PTimeStamp = GetTimestamp(line);
if (IsMeterEntry(line) && !meters.ContainsKey(MeterUID))
{
string MeterNr = GetMeterUID(line).Replace("4B414D", "");
int meternr = int.Parse(MeterNr, System.Globalization.NumberStyles.HexNumber);
meters.Add(MeterUID, new Meter()
{
MeterUID = MeterUID,
MeterNR = meternr,
P2Pmeterentry = new List<P2PMeterEntry>()
});
}
if (IsMeterEntry(line))
{
P2PMeterEntry p2pmeter = new P2PMeterEntry
{
P2PTimeStamp = P2PTimeStamp,
MimHashcode = MimHashcode,
FirmwareUploadStatus = FirmwareUploadStatus,
IsKnown = IsKnown,
P2PMetersession = new List<P2PMeterSession>()
};
if (IsNoLongerMeterEntry(line))
{
string SessionLevel = GetLevel(line);
string SessionMessage = GetSessionMessage(line);
string Context = GetSessionContext(line);
P2PMeterSession MeterSession = new P2PMeterSession
{
SessionTimeStamp = P2PTimeStamp,
SessionLevel = SessionLevel,
SessionMessage = SessionMessage,
Context = Context
};
meterSession.Add(MeterSession);
}
meters[MeterUID].P2Pmeterentry.Add(p2pmeter);
}
}
}
}
and the IsMeterEntry and IsNoLongerMeterEntry
//IsMeterSession
public static bool IsMeterEntry(string text)
{
return text.ToLower().Contains("announce message received:");
}
public static bool IsNoLongerMeterEntry(string text)
{
return text.ToLower().Contains("context - disposed");
}
Implement a simple state machine with two states: IgnoreLine (initial state) and Announce.
for each line in log
if line contains "Announce message"
read UID
create a StringBuilder
set state=Announce
else if line contains "Disposed"
store the StringBuilder's content in the dictionary[uid]
set state=IgnoreLine
else if state==Announce and line contains "certain words"
append line to StringBuilder