We Keep Coding

Table Row (//tr) HtmlNodes seem to have issue with rows being skipped

I'm currently trying to scrape a cannabis strain database as it is no longer being maintained. I seem to be running into an issue where table rows are skipped in my logic but it really doesn't make sense, it's like a break is being called when I'm iterating through the //tr elements of the table that is storing the chemical reports.
Is it something like the δ α symbols in the next row. I've tried regex replacing them out to now luck. Any help would be appreciated all code is in a single class console app.
Issue is in the table.ChildNodes not iterating all the way through. Located in the ParseChemical() method.
Sample Page: http://ocpdb.pythonanywhere.com/ocpdb/420/
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations.Schema;
using System.ComponentModel.Design;
using System.IO.IsolatedStorage;
using System.Linq;
using System.Linq.Expressions;
using System.Net;
using System.Net.Http;
using System.Reflection.Emit;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Microsoft.VisualBasic.CompilerServices;
namespace CScrape
{
class Program
{
private static readonly HttpClient Client = new HttpClient();
private static readonly string BaseUri = "http://ocpdb.pythonanywhere.com/ocpdb/";
private static int _startId = 420;
private static int _endId = 1519;
private static List<Lab> _labs = new List<Lab>();
private static List<ChemicalItem> _chemicalItems = new List<ChemicalItem>();
private static List<UnitOfMeasure> _uoms = new List<UnitOfMeasure>();
private static List<Strain> _strains = new List<Strain>();
static void Main(string[] args)
{
Client.DefaultRequestHeaders.Accept.Clear();
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls13;
_uoms.AddRange(GetUoms());
for (var i = _startId; i <= _endId; i++)
{
var result = GetUri($"{BaseUri}{i}").Result;
_strains.Add(ParseChemical(result));
}
}
private static long AddChemicalItem(ChemicalItem item)
{
if (ChemicalExists(item.Symbol))
return _chemicalItems.FirstOrDefault(ci => ci.Symbol == item.Symbol)?.Id ?? -1;
item.Id = _chemicalItems.Count + 1;
_chemicalItems.Add(item);
return item.Id;
}
private static void UpdateChemicalItem(ChemicalItem item)
{
if (!ChemicalExists(item.Symbol)) return;
var index = _chemicalItems.IndexOf(item);
if (!(index >= 0)) return;
_chemicalItems.RemoveAt(index);
AddChemicalItem(item);
}
private static long AddLab(Lab lab)
{
if (LabExists(lab.Name))
return _labs.FirstOrDefault(l => l.Name == lab.Name)?.Id ?? -1;
lab.Id = _labs.Count + 1;
_labs.Add(lab);
return lab.Id;
}
private static async Task<string> GetUri(string uri)
{
var response = await Client.GetByteArrayAsync(uri);
return Encoding.UTF8.GetString(response, 0, response.Length - 1);
}
private static Strain ParseChemical(string html)
{
html = Regex.Replace(html, #"Δ", "Delta");
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(html);
var strain = new Strain();
strain.Reports ??= new List<ChemicalReport>();
try
{
strain.Name = htmlDoc.DocumentNode.SelectSingleNode("/html/body/div/div[1]/div[1]/h3/b").InnerText;
}
catch (Exception e)
{
// TODO: DOcument Exception
Console.WriteLine(e.Message);
}
if (string.IsNullOrWhiteSpace(strain.Name)) return null;
try
{
var ocpId = htmlDoc.DocumentNode.SelectSingleNode("/html/body/div/div[1]/div[2]/p/b/text()[1]");
strain.OcpId = SanitizeHtml(ocpId.InnerText).Split(':')[1];
}
catch (Exception e)
{
// TODO: Document Exception
Console.WriteLine(e.Message);
}
if (string.IsNullOrWhiteSpace(strain.OcpId)) return null;
try
{
var date = htmlDoc.DocumentNode.SelectSingleNode("/html/body/div/div[1]/div[2]/p/text()");
}
catch (Exception e)
{
// TODO: Document Exception
Console.WriteLine(e.Message);
}
var chemReport = new ChemicalReport();
chemReport.Items ??= new List<ReportItem>();
try
{
var table = htmlDoc.DocumentNode.SelectSingleNode("/html/body/div/div[2]/div[1]/table/tbody");
var children = table.ChildNodes.ToList();
// On the sample page there are 200 children here
// However it only interates through the first few and then just breaks out of the loop
foreach (var child in children)
{
var name = child.Name;
if (child.Name == "tr")
{
var infos = child.SelectNodes("th|td");
foreach (var info in infos)
{
if(string.IsNullOrWhiteSpace(info.InnerText)) continue;
if (info.InnerText.Contains("Report")) continue;
if (double.TryParse(info.InnerText, out var isNumber))
{
var last = chemReport.Items.LastOrDefault();
if (last == null) continue;
if (last.Value <= 0.0000) last.Value = isNumber;
else
{
var further = chemReport.Items.ToArray()[chemReport.Items.Count - 2];
if (further.Value <= 0.0000)
further.Value = isNumber;
}
continue;
}
var _ = new ChemicalItem
{
Name = info.InnerText,
Symbol = info.InnerText
};
_.Id = AddChemicalItem(_);
var report = new ReportItem
{
Chemical = _,
ChemicalItemId = _.Id,
UnitOfMeasureId = 1,
UoM = GetUoms()[0]
};
chemReport.Items.Add(report);
}
}
}
strain.Reports.Add(chemReport);
}
catch (Exception e)
{
// TODO: Document exception
Console.Write(e.Message);
}
return strain;
}
private static List<UnitOfMeasure> GetUoms()
{
return new List<UnitOfMeasure>
{
new UnitOfMeasure {Name = "Milligrams per Gram", Symbol = "mg/g"},
new UnitOfMeasure {Name = "Percent", Symbol = "%"}
};
}
private static string SanitizeHtml(string text, string replacement = "")
{
return Regex.Replace(text, #"<[^>]+>| |α|\n|\t", replacement);
}
private static string GetLabName(string[] split)
{
var strip = split[0].Split(':')[1];
for(var i = 1; i < split.Length - 2; i ++)
{
if (string.IsNullOrWhiteSpace(split[i])) break;
strip += $" {split[i]}";
}
return strip;
}
private static string GetSampleId(string[] split)
{
var found = false;
foreach (var item in split)
{
if (found)
return item.Split(':')[1];
if (item == "Sample") found = true;
}
return "NA";
}
private static bool LabExists(string name)
{
return _labs.Any(lab => lab.Name == name);
}
private static bool ChemicalExists(string name)
{
return _chemicalItems.Any(ci => ci.Symbol == name);
}
private static ReportItem GetReportItem(string text)
{
if (string.IsNullOrWhiteSpace(text)) return null;
ReportItem ri = null;
try
{
var clean = SanitizeHtml(text);
var check = 0;
var split = clean.Split(':');
var label = split[0];
if (string.IsNullOrWhiteSpace(label)) return null;
if (double.TryParse(label, out var invalidType)) return null;
var val = string.Empty;
if (split.Length == 1)
{
if (split[0].Contains("Total"))
{
Regex re = new Regex(#"([a-zA-Z]+)(\d+)");
Match result = re.Match(split[0]);
label = result.Groups[1].Value;
val = result.Groups[2].Value;
}
}
if(split.Length > 1)
val = split[1];
if (!ChemicalExists(label)) AddChemicalItem(new ChemicalItem {Id = _chemicalItems.Count + 1,Symbol = label});
ri = new ReportItem();
ri.Chemical = _chemicalItems.FirstOrDefault(ci => ci.Symbol == label);
ri.UoM = val.Contains("%")
? _uoms.FirstOrDefault(uom => uom.Symbol == "%")
: _uoms.FirstOrDefault(uom => uom.Symbol == "mg/g");
if (string.IsNullOrWhiteSpace(val)) return ri;
var value = val.Contains("%") ? split[1].Substring(0, val.Length - 1) : val;
ri.Value = Convert.ToDouble(value);
}
catch (Exception e)
{
// TODO: Document Exception
Console.WriteLine(e.Message);
}
return ri;
}
//private static ChemicalItem GetChemicalItem(string text)
//{
//}
public class Strain
{
public long Id { get; set; }
public string Name { get; set; }
public DateTime Created { get; set; }
public string OcpId { get; set; }
public bool IsHidden { get; set; } = false;
public virtual ICollection<ChemicalReport> Reports { get; set; }
}
public class Lab
{
public long Id { get; set; }
public string Name { get; set; }
public virtual ICollection<ChemicalReport> Reports { get; set; }
}
public class ChemicalReport
{
public long Id { get; set; }
[ForeignKey("Lab")]
public long LabId { get; set; }
public virtual Lab Lab { get; set; }
public string SampleId { get; set; }
public DateTime Created { get; set; }
public virtual ICollection<ReportItem> Items { get; set; }
[ForeignKey("Strain")]
public long StrainId { get; set; }
public virtual Strain Strain { get; set; }
}
public class ChemicalItem
{
public long Id { get; set; }
public string Name { get; set; }
public string Symbol { get; set; }
}
public class ReportItem
{
public long Id { get; set; }
[ForeignKey("Chemical")]
public long ChemicalItemId { get; set; }
public virtual ChemicalItem Chemical { get; set; }
public double Value { get; set; }
[ForeignKey("UoM")]
public long UnitOfMeasureId { get; set; }
public virtual UnitOfMeasure UoM { get; set; }
}
public class UnitOfMeasure
{
public long Id { get; set; }
public string Name { get; set; }
public string Description { get; set; }
public string Symbol { get; set; }
}
}
}

Recursively create a tree hierarchy off an array of strings

I have an array of strings separated by "!". I am trying to break that string up and create a tree hierarchy recursively in my custom class called PivotGroup. For example, what I am aiming at is to break up string array
string[] paths = new string[] {
"ROOT!ZZZ!AAA!EEE!15712",
"ROOT!ZZZ!AAA!EEE!15722",
"ROOT!ZZZ!AAA!EEE!13891"}
Into the PivotGroup class such as PivotGroup contains ChildGroups[] that embed the array strings.
So for example:
PivotGroup pgGroup = new PivotGroup();
pgGroup.ChildGroups[0] = PivotGroup[]; // Key:Book Level 3 Value: "AAA"
Now within Book Level 3 ChildGroups I need to set Book Level 4 which value is "EEE" and within the ChildGroups of "EEE" I would need to create another childGroup array which size in the case would be 3 called Book Level 5 and set another PivotGroup for each of following 15712, 15722, 13891
Here is my PivotGroup Class and embedded class Objects:
public class PivotGroup
{
public PivotGroup() { }
public PivotGroup(PivotGroupKey groupKey, PivotRow data, PivotGroup[] childGroups, bool leaf, int groupLevel)
{
GroupKey = groupKey;
Data = data;
ChildGroups = childGroups;
Leaf = leaf;
GroupLevel = groupLevel;
}
public PivotGroupKey GroupKey { get; private set; }
public PivotRow Data { get; private set; }
public PivotGroup[] ChildGroups { get; set; }
public bool Leaf { get; private set; }
public int GroupLevel { get; private set; }
public override string ToString()
{
return GroupKey + ", GroupLevel: " + GroupLevel + ", Children: " +
ChildGroups.Length + (Leaf ? " (Leaf)" : "");
}
}
public class PivotGroupKey
{
public PivotGroupKey()
{
}
public PivotGroupKey(string keyGroup, string keyValue)
{
if(keyGroup != null)
KeyGroup = string.Intern(keyGroup);
if (keyValue != null)
KeyValue = string.Intern(keyValue);
}
public string KeyGroup { get; private set; }
public string KeyValue { get; private set; }
public override string ToString()
{
return KeyGroup + ": " + KeyValue;
}
}
public class PivotRow
{
public PivotRow()
{
}
public PivotRow(string key, params object[] data) : this(key, true, data) { }
public PivotRow(string key, bool entitled, params object[] data)
{
Data = data;
Key = null;
Entitled = entitled;
}
public object[] Data { get; private set; }
public bool Entitled { get; private set; }
public string Key { get { return null; } set { } }
}
Main program I tried:
public class BookLevels
{
public string Root { get; set; }
public string BookLevel2 { get; set; }
public string BookLevel3 { get; set; }
public string BookLevel4 { get; set; }
public string BookLevel5 { get; set; }
}
class Program
{
static void BuildTree(string[] paths)
{
var BookPaths = paths.Select(x => x.Split('!'))
.Select(x => new BookLevels
{
Root = x[0],
BookLevel2 = x[1],
BookLevel3 = x[2],
BookLevel4 = x[3],
BookLevel5 = x[4]
}).GroupBy(z => new { z.BookLevel3, z.BookLevel4 }).ToArray();
var BookLevel3Cnt = BookPaths.Select(q => q.Key.BookLevel3).Count();
PivotGroup root = new PivotGroup(
new PivotGroupKey("Total", ""),
new PivotRow(null, new string[8]),
new PivotGroup[BookLevel3Cnt], false, 0);
foreach (var booklevel3 in BookPaths)
{
AddChildren(root, booklevel3);
}
}
private static void AddChildren(PivotGroup root, IGrouping<object, BookLevels> booklevel, int index = 0)
{
root.ChildGroups[index] = new PivotGroup(
new PivotGroupKey("Book Level " + (index + 3).ToString(), booklevel.Key.ToString()),
new PivotRow(null, new string[8]),
AddChildren(root, booklevel[index], index + 1), false, 0);
}
static void Main(string[] args)
{
string[] paths = new string[] {
"ROOT!ZZZ!AAA!EEE!15712",
"ROOT!ZZZ!AAA!EEE!15722",
"ROOT!ZZZ!AAA!EEE!13891",
"ROOT!ZZZ!AAA!DDD!15712",
"ROOT!ZZZ!AAA!DDD!15722",
"ROOT!ZZZ!AAA!DDD!13891",
"ROOT!ZZZ!BBB!DDD!15812",
"ROOT!ZZZ!BBB!DDD!15822",
"ROOT!ZZZ!BBB!DDD!13891",
};
BuildTree(paths);
Console.WriteLine();
Console.ReadLine();
}
I think my issue might be the way I am creating the Linq statement that breaks up the string, since I'm not sure how to progress thru it recursively.

I'm not sure what goes into which property. Also, for sake of simplicity and to be able to concentrate on the recursive algorithm, I redefine the group class like this (it does not mean that you have to change your class, instead, adapt my algorithm):
public class PivotGroup
{
public string Key { get; set; }
public List<PivotGroup> ChildGroups { get; } = new List<PivotGroup>();
public override string ToString() => Key; // Makes debugging easier.
}
The idea is that the values of the path go into the key. I made ChildGroups a list to be able to add children successively. My BuildTree returns the root
static PivotGroup BuildTree(string[] paths)
{
var root = new PivotGroup { Key = "ROOT" };
foreach (string path in paths) {
AddChildren(root, path.Split('!').Skip(1).ToList());
}
return root;
}
The recursive part goes into AddChildren. I convert the path into a List<string> to be able to remove the added part. AddChildren assumes that the first item in path is the first child to be added.
static void AddChildren(PivotGroup group, List<string> path)
{
string key = path[0];
int index = group.ChildGroups.FindIndex(g => g.Key == key);
PivotGroup child;
if (index >= 0) { // A child with this key exists.
child = group.ChildGroups[index]; // Select this existing child.
} else { // This key is missing. Add a new child.
child = new PivotGroup { Key = key };
group.ChildGroups.Add(child);
}
if (path.Count > 1) {
path.RemoveAt(0); // Remove the added child key and add the rest recursively.
AddChildren(child, path);
}
}
We add children by walking down the tree and adding new children if necessary.
This prints the tree recursively:
private static void PrintTree(PivotGroup group, int level)
{
Console.WriteLine(new String(' ', 2 * level) + group.Key);
foreach (PivotGroup child in group.ChildGroups) {
PrintTree(child, level + 1);
}
}
string[] paths = new string[] {
"ROOT!ZZZ!AAA!EEE!15712",
...
};
PivotGroup root = BuildTree(paths);
PrintTree(root, 0);
Console.ReadKey();
We could also use a loop instead of doing a recursion, since we add one branch at a time:
static PivotGroup BuildTree(string[] paths)
{
var root = new PivotGroup { Key = "ROOT" };
foreach (string path in paths) {
PivotGroup group = root;
string[] pathElements = path.Split('!');
for (int i = 1; i < pathElements.Length; i++) { // Element [0] is ROOT, we skip it.
string key = pathElements[i];
int index = group.ChildGroups.FindIndex(g => g.Key == key);
PivotGroup child;
if (index >= 0) { // A child with this key exists.
child = group.ChildGroups[index]; // Select this existing child.
} else { // This key is missing. Add a new child.
child = new PivotGroup { Key = key };
group.ChildGroups.Add(child);
}
group = child;
}
}
return root;
}
List<T>.FindIndex is inefficient for large lists. If you have large data sets and the order does not matter, switch to Dictionary<string, PivotGroup>. If you need the data to be sorted, use SortedDictionary<string, PivotGroup>.

Here is some simple recursive code :
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace ConsoleApplication1
{
class Program
{
static void Main(string[] args)
{
string[] paths = new string[] {
"ROOT!ZZZ!AAA!EEE!15712",
"ROOT!ZZZ!AAA!EEE!15722",
"ROOT!ZZZ!AAA!EEE!13891"};
List<List<string>> inputData = paths.Select(x => x.Split(new char[] {'!'}).ToList()).ToList();
Node root = new Node();
Node.ParseTree(root, inputData);
}
}
public class Node
{
public string name { get; set; }
public List<Node> children { get; set; }
public static void ParseTree(Node parent, List<List<string>> inputData)
{
parent.name = inputData.First().FirstOrDefault();
var groups = inputData.Select(x => x.Skip(1)).GroupBy(x => x.Take(1).FirstOrDefault());
foreach (var group in groups)
{
if (group.Key != null)
{
if (parent.children == null) parent.children = new List<Node>();
Node newNode = new Node();
parent.children.Add(newNode);
ParseTree(newNode, group.Select(x => x.Select(y => y).ToList()).ToList());
}
}
}
}
}

C# Parse items from namespace

I have the following XML:
https://pastebin.com/YQBhNzm5
I want to match up the item values with the field values.
XmlDocument xdoc = new XmlDocument();
xdoc.Load(ofd.FileName);
XmlNamespaceManager xmanager = new XmlNamespaceManager(xdoc.NameTable);
xmanager.AddNamespace("ns", "http://www.canto.com/ns/Export/1.0");
var result = xdoc.SelectNodes("//ns:Layout/ns:Fields", xmanager);
foreach(XmlElement item in result)
{
Console.WriteLine(item.InnerText);
}
When I do this I get all the field names in one line. How can I iterate through all fields in layout and go one by one?

I parsed xml using xml linq. First I put items into a dictionary. Then I parsed the fields looking up the uid from the dictionary. I parsed the fields recursively to keep the hierarchy.
It looks like the uid, type, value, and name are always the same for each item, but an item can appear in multiple catalogs with a catalog id and an id.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Xml;
using System.Xml.Linq;
namespace ConsoleApplication1
{
class Program
{
const string FILENAME = #"c:\temp\test.xml";
static void Main(string[] args)
{
Layout layout = new Layout(FILENAME);
}
}
public class Layout
{
public string tablename { get; set; }
public List<Field> fields { get; set; }
public Layout layout { get; set; }
public Dictionary<string, Item> dict = new Dictionary<string, Item>();
public Layout() { }
public Layout(string filename)
{
XDocument doc = XDocument.Load(filename);
XElement xLayout = doc.Descendants().Where(x => x.Name.LocalName == "Layout").FirstOrDefault();
XNamespace ns = xLayout.GetNamespaceOfPrefix("ns");
foreach (XElement item in doc.Descendants(ns + "Item"))
{
int catalogid = (int)item.Attribute("catalogid");
int id = (int)item.Attribute("id");
foreach(XElement fieldValue in item.Elements(ns + "FieldValue"))
{
string uid = (string)fieldValue.Attribute("uid");
uid = uid.Replace("{", "");
uid = uid.Replace("}", "");
string innertext = (string)fieldValue;
string displayValue = (string)fieldValue.Attribute("displayValue");
List<string> categoryValues = fieldValue.Elements(ns + "CategoryValue").Select(x => (string)x).ToList();
if (!dict.ContainsKey(uid))
{
Item newItem = new Item() {
catalogidId = new List<KeyValuePair<int, int>>() {new KeyValuePair<int, int>(catalogid, id)},
innertext = innertext,
uid = uid,
displayValue = displayValue,
categoryValues = categoryValues
};
dict.Add(uid, newItem);
}
else
{
dict[uid].catalogidId.Add(new KeyValuePair<int, int>(catalogid, id));
}
}
}
layout = new Layout();
RecursiveParse(ns, xLayout, layout);
}
public void RecursiveParse(XNamespace ns, XElement parent, Layout layout)
{
layout.tablename = (string)parent.Attribute("tableName");
foreach(XElement xField in parent.Element(ns + "Fields").Elements(ns + "Field"))
{
if (layout.fields == null) layout.fields = new List<Field>();
Field newField = new Field();
layout.fields.Add(newField);
newField.uid = (string)xField.Attribute("uid");
newField.uid = newField.uid.Replace("{", "");
newField.uid = newField.uid.Replace("}", "");
newField._type = (int)xField.Attribute("type");
newField.value = (int)xField.Attribute("valueInterpretation");
newField.name = (string)xField.Element(ns + "Name");
if (dict.ContainsKey(newField.uid))
{
newField.items = dict[newField.uid];
}
if (xField.Element(ns + "Layout") != null)
{
Layout newLayout = new Layout();
newField.layout = newLayout;
RecursiveParse(ns, xField.Element(ns + "Layout"), newLayout);
}
}
}
public class Field
{
public string uid { get; set; }
public int _type { get; set; }
public int value { get; set; }
public string name { get; set; }
public Layout layout { get; set; }
public Item items { get; set; }
}
public class Item
{
public List<KeyValuePair<int, int>> catalogidId { get; set; }
public string uid { get; set; }
public string innertext { get; set; }
public string displayValue { get; set; }
public List<string> categoryValues { get; set; }
}
}
}

Form a tree of files and folders

How can I recursively get all folders and files tree from folder? I need to put this tree into DataTable and form a treeView from it. I have a little problem with forming recursion methods, for now I only have a draft which doesn't work properly:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Controls.Primitives;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;
namespace SWPF
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
private int id, parentID;
private List<TREE> tree;
public MainWindow()
{
InitializeComponent();
tree = new List<TREE>();
id = 0;
parentID = 0;
fill(#"D:\c#");
}
private List<TREE> fill(string path)
{
DirectoryInfo dir = new DirectoryInfo(path);
foreach (DirectoryInfo folder in dir.GetDirectories())
{
tree.Add(new TREE(){ID = id++, ParentID = parentID++, Name = folder.Name, Type = "Folder"});
foreach (FileInfo file in folder.GetFiles())
{
tree.Add(new TREE() { ID = id, ParentID = parentID, Name = file.Name, Type = "File" });
}
var p = folder.GetDirectories();
if (p.Length > 0)
{
dir = folder;
fill(dir.FullName);
}
}
return tree;
}
}
public class TREE
{
public int ID { get; set; }
public int ParentID {get;set;}
public string Name {get;set;}
public string Type {get;set;}
}
}
Please, could you show me right way forming and putting tree into DataTable. List is good, but I need DataTable.
I tried this, but it doesn't want to work properly:
public DataTable fill(DirectoryInfo dir)
{
DataTable dt = new DataTable("Tasks");
//dt.Columns.Add("ID");
dt.Columns.Add("ParentID");
dt.Columns.Add("Name");
dt.Columns.Add("Type");
foreach (var folder in dir.GetDirectories())
{
var row = dt.NewRow();
row["ParentID"] = folder.Parent;
row["Name"] = folder.Name;
row["Type"] = "dir";
dt.Rows.Add(row);
foreach (var file in folder.GetFiles())
{
var row2 = dt.NewRow();
row["ParentID"] = folder.Parent;
row["Name"] = file.Name;
row["Type"] = "file";
dt.Rows.Add(row2);
}
fill(folder);
}
return dt;
}

Here's how I'd implement the recursive part:
static class IdGenerator
{
static int _id = 1;
public static int Generate() { return _id++; }
}
class Tree
{
public int Id { get; set; }
public string Name { get; set; }
public int ParentId { get; set; }
public Tree Parent { get; set; }
public virtual ICollection<string> Files { get; set; }
public virtual ICollection<Tree> Trees { get; set; }
internal Tree ()
{
Id = IdGenerator.Generate();
Name = String.Empty;
ParentId = 0;
Parent = null;
Files = new List<string>();
Trees = new List<Tree>();
}
public override string ToString()
{
return String.Join("\n", new[] { Id + "\t" + Path.Combine(Parent != null ? Parent.Name : String.Empty, Name) }.Union(Files).Union(Trees.Select(t => t.ToString()))) + "\n";
}
public static Tree CreateTree(string path, Tree parent, string pattern)
{
try
{
var tree = new Tree
{
Parent = parent,
ParentId = parent != null ? parent.Id : 0,
Name = Path.GetFileName(path),
Files = Directory.GetFiles(path, pattern, SearchOption.TopDirectoryOnly).Select(f => Path.GetFileName(f)).ToArray(),
};
tree.Trees = Directory.GetDirectories(path, "*.*", SearchOption.TopDirectoryOnly).Select(d => CreateTree(d, tree, pattern)).ToArray();
return tree;
}
catch (UnauthorizedAccessException)
{ return new Tree(); }
}
}
If I were to continue with this, I'd use Entity Framework Code First to generate the database from the POCO Tree, and drop the static IdGenerator since EF would generate these database IDs for me.
EDIT
However, here's how to do it in a DataTable:
class TreeDataTable : DataTable
{
public TreeDataTable()
: this("trees")
{
}
public TreeDataTable(string tableName)
: base(tableName)
{
Columns.Add("Id");
Columns.Add("Name");
Columns.Add("ParentId");
Columns.Add("Type");
}
public void AddFile(int id, string name, int parent)
{
AddRow(id, name, parent, typeof(FileInfo));
}
public void AddTree(Tree tree)
{
AddRow(tree.Id, tree.Name, tree.ParentId, typeof(DirectoryInfo));
var table = tree.ToDataTable();
foreach (var r in table.Rows.Cast<DataRow>())
AddRow(Convert.ToInt32(r["Id"]), r["Name"].ToString(), Convert.ToInt32(r["ParentId"]), Type.GetType(r["Type"].ToString()));
}
public string ToXml()
{
using (var sw = new StringWriter())
{
using (var tw = new XmlTextWriter(sw))
{
tw.Formatting = Formatting.Indented;
tw.WriteStartDocument();
tw.WriteStartElement(TableName);
((IXmlSerializable)this).WriteXml(tw);
tw.WriteEndElement();
tw.WriteEndDocument();
}
return sw.ToString();
}
}
private void AddRow(int id, string name, int parent, Type type)
{
var row = NewRow();
row["Id"] = id;
row["Name"] = name;
row["ParentId"] = parent;
row["Type"] = type.ToString();
Rows.Add(row);
}
}
And add this method to Tree:
public DataTable ToDataTable()
{
var table = new TreeDataTable();
foreach (var f in Files)
table.AddFile(IdGenerator.Generate(), f, Id);
foreach (var t in Trees)
table.AddTree(t);
return table;
}
Here's how to use it:
class Program
{
static void Main(string[] args)
{
var path = args.FirstOrDefault(a => !String.IsNullOrEmpty(a)) ?? #"C:\Documents\Videos";
args = args.Skip(1).ToArray();
var pattern = args.FirstOrDefault(a => !String.IsNullOrEmpty(a)) ?? "*.mp4";
var tree = Tree.CreateTree(path, null, pattern);
Console.WriteLine((tree.ToDataTable() as TreeDataTable).ToXml());
}
}

how to create a collection from list of strings that represents a directory structure in C# or VB

First, I did check this post but it is in Python, first, and second it appears to be actually making the directories, which I cannot do in this scenario.
Second, these are not directories that exist, nor can I create them.
I have an input in C# like this:
List<string> filePaths = new List<string>();
filePaths.Add(#"ProgramDir\InstallDir\Module1\mod1pack1.exe");
filePaths.Add(#"ProgramDir\InstallDir\Module1\mod1pack2.exe");
filePaths.Add(#"ProgramDir\InstallDir\Module2\mod2pack1.exe");
filePaths.Add(#"ProgramDir\InstallDir\Module1\SubModule1\report1.rpt");
filePaths.Add(#"SystemDir\DependencyDir\dependency1.dll");
filePaths.Add(#"SystemDir\DependencyDir\dependency2.dll");
What I have been trying to do is create an object that represents this structure, such that it could be visualized like this:
-ProgramDir
Installdir
Module1
mod1pack1.exe
mod1pack2.exe
-SubModule1
report1.rpt
Module2
mod2pack1.exe
-SystemDir
-DependencyDir
dependency1.dll
dependency2.dll
What I have tried is various versions of the following, and I could really use some help to figure out where I've got it wrong.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace ConsoleApplication1
{
class Program
{
static void Main(string[] args)
{
SetFilePathList();
DTree forest = new DTree();
List<DTreeBranch> branches = new List<DTreeBranch>();
foreach (string path in filePaths)
{
forest.GrowTree(path.Split('\\'), branches);
}
forest.SubBranches.AddRange(branches);
}
private static List<string> filePaths { get; set; }
private static void SetFilePathList()
{
filePaths = new List<string>();
filePaths.Add(#"ProgramDir\InstallDir\Module1\mod1pack1.exe");
filePaths.Add(#"ProgramDir\InstallDir\Module1\mod1pack2.exe");
filePaths.Add(#"ProgramDir\InstallDir\Module2\mod2pack1.exe");
filePaths.Add(#"ProgramDir\InstallDir\Module1\SubModule1\report1.rpt");
filePaths.Add(#"SystemDir\DependencyDir\dependency1.dll");
filePaths.Add(#"SystemDir\DependencyDir\dependency2.dll");
}
}
public class DTree
{
public List<DTreeBranch> SubBranches { get; set; }
public string BranchName { get; set; }
public DTree() { SubBranches = new List<DTreeBranch>(); }
public DTreeBranch AddChildren(string[] childElements, DTreeBranch branch)
{
DTreeBranch childBranch;
foreach (string element in childElements)
{
childBranch = new DTreeBranch();
childBranch.BranchName = element;
branch.SubBranches.Add(childBranch);
var query = from q in childElements
where q != childBranch.BranchName
select q;
AddChildren(query.ToArray<string>(), childBranch);
}
return branch;
}
public void GrowTree(string[] pathElements, List<DTreeBranch> Branches)
{
DTreeBranch result = Branches.Find(delegate(DTreeBranch b)
{
return b.BranchName == pathElements[0];
});
if (result == null)
{
DTreeBranch newRootBranch = new DTreeBranch();
newRootBranch.BranchName = pathElements[0];
Branches.Add(newRootBranch);
GrowTree(pathElements, Branches);
}
else
{
var query = from q in pathElements
where q != result.BranchName
select q;
DTreeBranch childBranch = AddChildren(query.ToArray<string>(), result);
Branches.Add(childBranch);
}
}
}
public class DTreeBranch
{
public List<DTreeBranch> SubBranches { get; set; }
public string BranchName { get; set; }
public DTreeBranch()
{
SubBranches = new List<DTreeBranch>();
}
}
}
The main thing is that the output is only two layers deep. I guess what I'm saying is that the new elements are added to the depth, not the breadth, and I'm at a loss as to how to effectively work through this. I also think that I have way more code than I need.
Thanks in advance.

I'm not sure exactly what our goals are, but a simple recursive parse will do it quite easily. Wrote this up, and hope it helps. You can make it significantly more fancy if you want, with DTrees and sub branches, or separate collections for Files and Directories, etc. I don't really understand what all that code in there is for. If it has something to do with WIX, I'm sorry ;) And you could always use something like this to parse it out into the tree, and then convert that sanely to a different format.
this assumes no duplicate leaf nodes (file names).
if that isn't the case, just add a sanity check like for directories.
The main "Node" class -
public class Node
{
public string Name { get; set; }
public bool IsDirectory { get; set; }
public List<Node> Children = new List<Node>();
internal void AddChildren(string f)
{
var dirs = Path.GetDirectoryName(f);
if (string.IsNullOrEmpty(dirs))
{
// we are adding a file
var file = Path.GetFileName(f);
Children.Add(new Node {Name = file, IsDirectory = false});
}
else
{
// we are adding a directory
var firstDir = dirs.Split(Path.DirectorySeparatorChar)[0];
var childNode = Children.FirstOrDefault(d => d.Name == firstDir);
if (childNode == null)
{
childNode = new Node {Name = firstDir, IsDirectory = true};
Children.Add(childNode);
}
var subPath = f.Substring(firstDir.Length + 1);
childNode.AddChildren(subPath);
}
}
}
Calling it is simple, like this:
var filePaths = new List<string> {
#"ProgramDir\InstallDir\Module1\mod1pack1.exe",
#"ProgramDir\InstallDir\Module1\mod1pack2.exe",
#"ProgramDir\InstallDir\Module2\mod2pack1.exe",
#"ProgramDir\InstallDir\Module1\SubModule1\report1.rpt",
#"SystemDir\DependencyDir\dependency1.dll",
#"SystemDir\DependencyDir\dependency2.dll",
};
var node = new Node { Name = "Root", IsDirectory = true };
foreach (var f in filePaths )
{
node.AddChildren(f);
}
Printing it out (with indent per level, gives me this)
public static void PrintNode(Node node, int indent)
{
if (indent > 0) // don't print out root directory (level 1).
{
var ending = node.IsDirectory ? Path.DirectorySeparatorChar.ToString() : "*";
Console.WriteLine("{0}{1}{2}", new string('\t', indent - 1), node.Name, ending);
}
node.Children.ForEach(n => PrintNode(n, indent + 1));
}
ProgramDir\
InstallDir\
Module1\
mod1pack1.exe*
mod1pack2.exe*
SubModule1\
report1.rpt*
Module2\
mod2pack1.exe*
SystemDir\
DependencyDir\
dependency1.dll*
dependency2.dll*

I got about the same as Andrew:
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
namespace ConsoleApplication1
{
class Program
{
public static void Main(String[] args)
{
var filePaths = new List<string> {#"ProgramDir\InstallDir\Module1\mod1pack1.exe", #"ProgramDir\InstallDir\Module1\mod1pack2.exe", #"ProgramDir\InstallDir\Module2\mod2pack1.exe", #"ProgramDir\InstallDir\Module1\SubModule1\report1.rpt", #"SystemDir\DependencyDir\dependency1.dll", #"SystemDir\DependencyDir\dependency2.dll"};
var nodes = Parse(filePaths.ToArray());
foreach (var node in nodes)
Console.Out.WriteLine(node.ToString());
Console.ReadLine();
}
public static IEnumerable<Node> Parse(params String[] paths)
{
var roots = new NodeSet();
foreach (var path in paths)
{
var pathSplit = path.Split('\\');
Node current = null;
foreach (var pathElement in pathSplit)
{
var currentRoots = (current == null) ? roots : current.Children;
if (currentRoots.Contains(pathElement))
current = currentRoots[pathElement];
else
currentRoots.Add(current = new Node(pathElement));
}
}
return roots;
}
public class Node
{
public String Name { get; private set; }
public NodeSet Children { get; private set; }
public Node(String name)
{
if (String.IsNullOrWhiteSpace(name)) throw new ArgumentNullException("name");
Name = name;
Children = new NodeSet();
}
public override string ToString() { return ToString(1); }
private String ToString(Int32 indent)
{
var indentStr = Environment.NewLine + new string('\t', indent);
return Name + (Children.Count == 0 ? "" : indentStr + String.Join(indentStr, Children.Select(c => c.ToString(indent + 1)).ToArray()));
}
}
public class NodeSet : KeyedCollection<String, Node> {
protected override string GetKeyForItem(Node item) { return item.Name; }
}
}
}