We Keep Coding

Web scraping a listings website

I'm trying to scrape a website - ive accomplished this on other projects but i cant seem to get this right. It could be that ive been up for over 2 days working and maybe i am missing something. Please could someone look over my code? Here it is :
using System;
using System.Collections.Generic;
using HtmlAgilityPack;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Linq;
using System.Xml.Linq;
using System.IO;
public partial class _Default : System.Web.UI.Page
{
List<string> names = new List<string>();
List<string> address = new List<string>();
List<string> number = new List<string>();
protected void Page_Load(object sender, EventArgs e)
{
string url = "http://www.scoot.co.uk/find/" + "cafe" + " " + "-in-uk?page=" + "4";
var Webget = new HtmlWeb();
var doc = Webget.Load(url);
List<List<string>> mainList = new List<List<string>>();
foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//h2//a"))
{
names.Add(Regex.Replace(node.ChildNodes[0].InnerHtml, #"\s{2,}", " "));
}
foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//p[#class='result-address']"))
{
address.Add(Regex.Replace(node.ChildNodes[0].InnerHtml, #"\s{2,}", " "));
}
foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//p[#class='result-number']"))
{
number.Add(Regex.Replace(node.ChildNodes[0].InnerHtml, #"\s{2,}", " "));
}
XDocument doccy = new XDocument(
new XDeclaration("1.0", "utf-8", "yes"),
new XComment("Business For Sale"),
new XElement("Data",
from data in mainList
select new XElement("data", new XAttribute("data", "data"),
new XElement("Name : ", names[0]),
new XElement("Add : ", address[0]),
new XElement("Number : ", number[0])
)
)
);
var xml = doccy.ToString();
Response.ContentType = "text/xml"; //Must be 'text/xml'
Response.ContentEncoding = System.Text.Encoding.UTF8; //We'd like UTF-8
doccy.Save(Response.Output); //Save to the text-writer
}
}
The website lists business name, phone number and address and they are all defined by a class name (result-address, result-number etc). I am trying to get XML output so i can get the business name, address and phone number from each listing on page 4 for a presentation tomorrow but i cant get it to work at all!
The results are right in all 3 of the for each loops but they wont output in the xml i get an out of range error.

My first piece of advice would be to keep your CodeBehind as light as possible. If you bloat it up with business logic then the solution will become difficult to maintain. That's off topic, but I recommend looking up SOLID principles.
First, I've created a custom object to work with instead of using Lists of strings which have no way of knowing which address item links up with which name:
public class Listing
{
public string Name { get; set; }
public string Address { get; set; }
public string Number { get; set; }
}
Here is the heart of it, a class that does all the scraping and serializing (I've broken SOLID principles but sometimes you just want it to work right.)
using System.Collections.Generic;
using HtmlAgilityPack;
using System.IO;
using System.Xml;
using System.Xml.Serialization;
using System.Linq;
public class TheScraper
{
public List<Listing> DoTheScrape(int pageNumber)
{
List<Listing> result = new List<Listing>();
string url = "http://www.scoot.co.uk/find/" + "cafe" + " " + "-in-uk?page=" + pageNumber;
var Webget = new HtmlWeb();
var doc = Webget.Load(url);
// select top level node, this is the closest we can get to the elements in which all the listings are a child of.
var nodes = doc.DocumentNode.SelectNodes("//*[#id='list']/div/div/div/div");
// loop through each child
if (nodes != null)
{
foreach (var node in nodes)
{
Listing listing = new Listing();
// get each individual listing and manually check for nulls
// listing.Name = node.SelectSingleNode("./div/div/div/div/h2/a")?.InnerText; --easier way to null check if you can use null propagating operator
var nameNode = node.SelectSingleNode("./div/div/div/div/h2/a");
if (nameNode != null) listing.Name = nameNode.InnerText;
var addressNode = node.SelectSingleNode("./div/div/div/div/p[#class='result-address']");
if (addressNode != null) listing.Address = addressNode.InnerText.Trim();
var numberNode = node.SelectSingleNode("./div/div/div/div/p[#class='result-number']/a");
if (numberNode != null) listing.Number = numberNode.Attributes["data-visible-number"].Value;
result.Add(listing);
}
}
// filter out the nulls
result = result.Where(x => x.Name != null && x.Address != null && x.Number != null).ToList();
return result;
}
public string SerializeTheListings(List<Listing> listings)
{
var xmlSerializer = new XmlSerializer(typeof(List<Listing>));
using (var stringWriter = new StringWriter())
using (var xmlWriter = XmlWriter.Create(stringWriter, new XmlWriterSettings { Indent = true }))
{
xmlSerializer.Serialize(xmlWriter, listings);
return stringWriter.ToString();
}
}
}
Then your code behind would look something like this, plus references to the scraper class and model class:
public partial class _Default : System.Web.UI.Page
{
protected void Page_Load(object sender, EventArgs e)
{
TheScraper scraper = new TheScraper();
List<Listing> listings = new List<Listing>();
// quick hack to do a loop 5 times, to get all 5 pages. if this is being run frequently you'd want to automatically identify how many pages or start at page one and find / use link to next page.
for (int i = 0; i < 5; i++)
{
listings = listings.Union(scraper.DoTheScrape(i)).ToList();
}
string xmlListings = scraper.SerializeTheListings(listings);
}
}

Need advice on using XML as database C#

I am making an application that stores data that user inputs.
http://i827.photobucket.com/albums/zz200/ArnasG/question_in_stackowerflow__zps4f7uy3l7.png
I have a bit of an issue because of my lack of experience, I want to save all the data user inputs in XML file and to load it when program starts next time. I had an idea to use dataset to read all the data from XML file and then work with the table[0] of that dataset(add/delete rows). It turn out that I can not make it to work properly. It loads some blank lines and lines that I created in previous tries, but there is only two lines that are actually saved in XML file. How could I make this work?
Thank you for your time :)
Actual XML file:
http://i827.photobucket.com/albums/zz200/ArnasG/question_in_stackowerflow_V2_zpshmwjnllr.png
DataSet ListOfTrades = new DataSet();
DataTable Lentele = new DataTable();
ListOfTrades.Tables.Add(Lentele);
// adding columns to the table
try
{
DataColumn Pair = new DataColumn("Pair", typeof(string));
Pair.AllowDBNull = false;
DataColumn Entry = new DataColumn("Entry", typeof(string));
Entry.AllowDBNull = false;
DataColumn StopLoss = new DataColumn("StopLoss", typeof(string));
StopLoss.AllowDBNull = false;
DataColumn TakeProfit = new DataColumn("TakeProfit", typeof(string));
TakeProfit.AllowDBNull = false;
DataColumn TakeProfit1 = new DataColumn("TakeProfit1", typeof(string));
TakeProfit1.AllowDBNull = false;
DataColumn TakeProfit2 = new DataColumn("TakeProfit2", typeof(string));
TakeProfit2.AllowDBNull = false;
DataColumn TakeProfit3 = new DataColumn("TakeProfit3", typeof(string));
TakeProfit3.AllowDBNull = false;
DataColumn LongShort = new DataColumn("LongShort", typeof(string));
LongShort.AllowDBNull = false;
DataColumn WinLoss = new DataColumn("WinLoss", typeof(string));
WinLoss.AllowDBNull = false;
data.Tables[0].Columns.AddRange(new DataColumn[] {
Pair, Entry, StopLoss, TakeProfit, TakeProfit1, TakeProfit2,
TakeProfit3, LongShort, WinLoss
});
}
catch(Exception Ex)
{
MessageBox.Show(Ex.Message);
}
// Adding new line to the table after user clicks save button
private void button1_Click(object sender, EventArgs e)
{
DataRow eilute = ListOfTrades.Tables[0].NewRow();
eilute[0] = comboBox1.Text.ToString();
eilute[1] = textBox1.Text.ToString();
eilute[2] = textBox2.Text.ToString();
eilute[3] = textBox3.Text.ToString();
eilute[4] = textBox4.Text.ToString();
eilute[5] = textBox5.Text.ToString();
eilute[6] = textBox6.Text.ToString();
if (radioButton1.Checked) { eilute[7] = "Long"; }
else { eilute[7] = "short"; }
if (radioButton1.Checked) { eilute[8] = "Win"; }
else { eilute[8] = "Loss"; }
ListOfTrades.Tables[0].Rows.Add(eilute);
ListOfTrades.Tables[0].WriteXml(DefaultPathToJournalXML);
dataGridView1.Update();
dataGridView1.Refresh();
}

Not getting duplicated. here is the xml
<?xml version="1.0" standalone="yes"?>
<NewDataSet>
<Table1>
<Pair>AUD/USD</Pair>
<Entry>0.00000</Entry>
<StopLoss>0.00000</StopLoss>
<TakeProfit>0.00000</TakeProfit>
<TakeProfit1>0.00000</TakeProfit1>
<TakeProfit2>0.00000</TakeProfit2>
<TakeProfit3>0.00000</TakeProfit3>
<LongShort>short</LongShort>
<WinLoss>loss</WinLoss>
</Table1>
<Table1>
<Pair>AUD/USD</Pair>
<Entry>0.00000</Entry>
<StopLoss>0.00000</StopLoss>
<TakeProfit>0.00000</TakeProfit>
<TakeProfit1>0.00000</TakeProfit1>
<TakeProfit2>0.00000</TakeProfit2>
<TakeProfit3>0.00000</TakeProfit3>
<LongShort>short</LongShort>
<WinLoss>Loss</WinLoss>
</Table1>
</NewDataSet>
​
Here is code
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data;
namespace ConsoleApplication1
{
class Program
{
const string FILENAME = #"c:\temp\test.xml";
static void Main(string[] args)
{
DataSet ds = new DataSet();
ds.ReadXml(FILENAME);
}
}
}
​

Think object oriented, and think Linq.
For example, let's say you have this XML file:
<trades>
<trade>
<pair>some pair</pair>
<stop-loss>stop loss 1</stop-loss>
</trade>
<trade>
<pair>other pair</pair>
<stop-loss>stop loss 2</stop-loss>
</trade>
</trades>
You can create a Trade class to hold the data in the trade tags and then use a Linq query to populate the object given the XML file:
class Trade
{
public string Pair;
public string StopLoss;
// Other variables from trade tag would go here...
// Function that can load trade objects from XML file into a list of Trade objects (List<Trade>)
public static List<Trade> loadTrade(string xmlFilePath)
{
// Load your XML document given the path to the .xml file
var doc = XDocument.Load(xmlFilePath);
// For each trade element in the trades element
var trades = (from trade in doc.Element("trades").Elements("trade")
select new Trade
{
// For each element in the trade element, put value in class variable
Pair = trade.Element("pair").Value,
StopLoss = trade.Element("stop-loss").Value
}).ToList<Trade>();
return trades;
}
}
When you're ready to save to a file you basically do the opposite of the Linq query to create an XML file. It will look very similar.
On there other hand, read this article and consider if there's a better alternative.

Webservice Returns Data All On One Line

So I have a basic web service that is grabbing data from a MSSQL database. It works as it's supposed to in terms of SQL connections and queries however, I would like to know if it's possible to have the data displayed a little more elegantly. My goal is to return one name on each line but the web service just spits it all out between string tags and all on one line. I'm EXTREMELY new to C# and ASP.NET and am amazed I made it this far. I just need to format the data in a more pleasant way if possible.
Here is my Code
namespace CustomerService
{
/// <summary>
/// Summary description for Service1
/// </summary>
[WebService(Namespace = "http://tempuri.org/")]
[WebServiceBinding(ConformsTo = WsiProfiles.BasicProfile1_1)]
[System.ComponentModel.ToolboxItem(false)]
// To allow this Web Service to be called from script, using ASP.NET AJAX, uncomment the following line.
// [System.Web.Script.Services.ScriptService]
public class Service1 : System.Web.Services.WebService
{
[WebMethod]
//create new webMethod to get names
public string getNames()
{
int size = DataHelper.name().Count();
string[] names = new string[size];
names = DataHelper.name();
return toString(names);
}
//take array passed in and convert to one single string
public string toString(string[] names)
{
int size = names.Count();
string[] nameArray = new string[size];
nameArray = names;
string output = "";
for (int x = 0; x < size; x++)
{
output = output + "\n" + nameArray[x];
}
return output;
}
}
}
And then the DataHelper service:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Data;
using System.Data.SqlClient;
namespace CustomerService
{
public class DataHelper
{
//create method to get names from customer DB
public static string[] name()
{
string currentName ="";
string[] names = new string[100];
double checkingBal;
double savingsBal;
double cdBal;
double mmBal;
//create connection
SqlConnection conn = new SqlConnection(#"Data Source=STE2074;Initial Catalog=ATMWeb;Persist Security Info=True;User ID=stp;Password=stp48329");
//create command to get names
string sqlString = "SELECT * FROM tblUsers";
SqlCommand cmd = new SqlCommand(sqlString, conn);
conn.Open();
int x = 0;
using (SqlDataReader reader = cmd.ExecuteReader())
{
while (reader.Read())
{
currentName = reader["firstName"].ToString();
names[x] = currentName;
x++;
}
//close connections
conn.Close();
reader.Close();
return names;
}
}
}
And this is the final result:
<?xml version="1.0" encoding="UTF-8"?>
<string xmlns="http://tempuri.org/"> Stephen lisa steve kyle s Lisa steven Customer chelsea jon karen jessica meagan </string>
I would like a new line after each name...is this possible?

Create a class and return that.
PUBLIC Class MyOldStringNames
{
property string NameHolder { get; set; }
}
End Class
List<MyOldStringNames> myListOfOldStringNames = new List<MyOldStringNames>();
For Each item in names
MyOldStringNames myItemToAdd = new MyOldStringNames() { NameHolder = item.toString() };
myListOfOldStringNames.add(myItemToAdd);
End
return myListOfOldStringNames;
Of course your method would then be something like this. :
public List<MyOldStringNames> getNames()
{
int size = DataHelper.name().Count();
string[] names = new string[size];
names = DataHelper.name();
//Do stuff above
//return data as shown above;
}

Loading a DataTable with a XML string

I have the XMLDocument and I extract the following xml tags I want using the xmlDocument.SelectSingleNode("./tag") into a string and I want to load it inside a DataTable.
I tried using the dataTable.ReadXML(); but the overloads for this function do not allow a string argument.
Is there a better way of doing this?
Edit : Adding Code
XmlDocument xdoc = new XmlDocument();
xdoc.LoadXml(string_With_Xml);
DataTable accessTable = new DataTable();
accessTable.ReadXml();
I Hope this adds more context to the question.

You can try the following:
//Your xml
string TestSTring = #"<Contacts>
<Node>
<ID>123</ID>
<Name>ABC</Name>
</Node>
<Node>
<ID>124</ID>
<Name>DEF</Name>
</Node>
</Contacts>";
StringReader StringStream = new StringReader(TestSTring);
DataSet ds = new DataSet();
ds.ReadXml(StringStream);
DataTable dt = ds.Tables[0];

you can write extension like this:
public static someType ReadXml(this DataTable dt, string yourParam1, string yourParam2)
{
method body....
}

You can do the following approach, the byte array here can be loaded for a string using for example:
Encoding.UTF8.GetBytes(somestring)
Helper method to load the datatable, note fallback to DataSet's method ReadXml instead of Datatable ReadXml. This will not always be suitable in case your xml contains somehow several data tables, as this method always returns the first data table in the catch:
public DataTable Convert(byte[] bytes)
{
var text = bytes.ToStringUtf8();
if (string.IsNullOrWhiteSpace(text))
{
return null;
}
using (var stream = new MemoryStream(bytes))
{
try
{
var dt = new DataTable();
dt.ReadXml(stream);
return dt;
}
catch (InvalidOperationException ie)
{
Trace.WriteLine(ie);
var ds = new DataSet();
stream.Position = 0;
ds.ReadXml(stream);
if (ds.Tables.Count > 0)
{
return ds.Tables[0];
}
return null;
}
}
}

Combining two SyndicationFeeds

What's a simple way to combine feed and feed2? I want the items from feed2 to be added to feed. Also I want to avoid duplicates as feed might already have items when a question is tagged with both WPF and Silverlight.
Uri feedUri = new Uri("http://stackoverflow.com/feeds/tag/silverlight");
XmlReader reader = XmlReader.Create(feedUri.AbsoluteUri);
SyndicationFeed feed = SyndicationFeed.Load(reader);
Uri feed2Uri = new Uri("http://stackoverflow.com/feeds/tag/wpf");
XmlReader reader2 = XmlReader.Create(feed2Uri.AbsoluteUri);
SyndicationFeed feed2 = SyndicationFeed.Load(reader2);

You can use LINQ to simplify the code to join two lists (don't forget to put System.Linq in your usings and if necessary reference System.Core in your project) Here's a Main that does the union and prints them to console (with proper cleanup of the Reader).
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using System.ServiceModel.Syndication;
namespace FeedUnion
{
class Program
{
static void Main(string[] args)
{
Uri feedUri = new Uri("http://stackoverflow.com/feeds/tag/silverlight");
SyndicationFeed feed;
SyndicationFeed feed2;
using(XmlReader reader = XmlReader.Create(feedUri.AbsoluteUri))
{
feed= SyndicationFeed.Load(reader);
}
Uri feed2Uri = new Uri("http://stackoverflow.com/feeds/tag/wpf");
using (XmlReader reader2 = XmlReader.Create(feed2Uri.AbsoluteUri))
{
feed2 = SyndicationFeed.Load(reader2);
}
SyndicationFeed feed3 = new SyndicationFeed(feed.Items.Union(feed2.Items));
StringBuilder builder = new StringBuilder();
using (XmlWriter writer = XmlWriter.Create(builder))
{
feed3.SaveAsRss20(writer);
System.Console.Write(builder.ToString());
System.Console.Read();
}
}
}
}

Well, one possibility is to create a new syndication feed that is a clone of the first feed, and then simply iterate through each post on the second one, check the first for its existence, and add it if it doesn't exist.
Something along the lines of:
SyndicationFeed newFeed = feed.clone;
foreach(SyndicationItem item in feed2.items)
{
if (!newFeed.contains(item))
newFeed.items.Add(item);
}
might be able to do it. It looks like 'items' is a simple enumberable list of syndication items, so theres not reason you can't simply add them.

If it's solely for stackoverflow, you can use this :
https://stackoverflow.com/feeds/tag/silverlight%20wpf
This will do an union of the two tags.
For a more general solution, I don't know. You'd probably have to manually iterate the elements of the two feeds and join them together. You can compare the <id> elements of <entry>s to see if they are duplicates.

I've turned today's accepted answer into a unit test just to explore this slightly:
[TestMethod]
public void ShouldCombineRssFeeds()
{
//reference: http://stackoverflow.com/questions/79197/combining-two-syndicationfeeds
SyndicationFeed feed;
SyndicationFeed feed2;
var feedUri = new Uri("http://stackoverflow.com/feeds/tag/silverlight");
using(var reader = XmlReader.Create(feedUri.AbsoluteUri))
{
feed = SyndicationFeed.Load(reader);
}
Assert.IsTrue(feed.Items.Count() > 0, "The expected feed items are not here.");
var feed2Uri = new Uri("http://stackoverflow.com/feeds/tag/wpf");
using(var reader2 = XmlReader.Create(feed2Uri.AbsoluteUri))
{
feed2 = SyndicationFeed.Load(reader2);
}
Assert.IsTrue(feed2.Items.Count() > 0, "The expected feed items are not here.");
var feedsCombined = new SyndicationFeed(feed.Items.Union(feed2.Items));
Assert.IsTrue(
feedsCombined.Items.Count() == feed.Items.Count() + feed2.Items.Count(),
"The expected number of combined feed items are not here.");
var builder = new StringBuilder();
using(var writer = XmlWriter.Create(builder))
{
feedsCombined.SaveAsRss20(writer);
writer.Flush();
writer.Close();
}
var xmlString = builder.ToString();
Assert.IsTrue(new Func<bool>(
() =>
{
var test = false;
var xDoc = XDocument.Parse(xmlString);
var count = xDoc.Root.Element("channel").Elements("item").Count();
test = (count == feedsCombined.Items.Count());
return test;
}
).Invoke(), "The expected number of RSS items are not here.");
}

//Executed and Tested :)
using (XmlReader reader = XmlReader.Create(strFeed))
{
rssData = SyndicationFeed.Load(reader);
model.BlogFeed = rssData; ;
}
using (XmlReader reader = XmlReader.Create(strFeed1))
{
rssData1 = SyndicationFeed.Load(reader);
model.BlogFeed = rssData1;
}
SyndicationFeed feed3 = new SyndicationFeed(rssData.Items.Union(rssData1.Items));
model.BlogFeed = feed3;
return View(model);

This worked fine for me:
// create temporary List of SyndicationItem's
List<SyndicationItem> tempItems = new List<SyndicationItem>();
// add all feed items to the list
tempItems.AddRange(feed.Items);
tempItems.AddRange(feed2.Items);
// remove duplicates with Linq 'Distinct()'-method depending on yourattributes
// add list without duplicates to 'feed2'
feed2.Items = tempItems