We Keep Coding

Problems to extract text from PDF for certain pdfs only C#

I need to extract some data from a PDF file.
I'm using the iTextSharp to do that.
I'm using this code which I founded on the net:
using System;
using System.IO;
using iTextSharp.text.pdf;
namespace PdfToText
{
/// <summary>
/// Parses a PDF file and extracts the text from it.
/// </summary>
public class PDFParser
{
/// BT = Beginning of a text object operator
/// ET = End of a text object operator
/// Td move to the start of next line
/// 5 Ts = superscript
/// -5 Ts = subscript
#region Fields
#region _numberOfCharsToKeep
/// <summary>
/// The number of characters to keep, when extracting text.
/// </summary>
private static int _numberOfCharsToKeep = 15;
#endregion
#endregion
#region ExtractText
/// <summary>
/// Extracts a text from a PDF file.
/// </summary>
/// <param name="inFileName">the full path to the pdf file.</param>
/// <param name="outFileName">the output file name.</param>
/// <returns>the extracted text</returns>
public bool ExtractText(string inFileName, string outFileName)
{
StreamWriter outFile = null;
try
{
outFileName = String.Empty;
outFileName = Path.GetDirectoryName(System.AppDomain.CurrentDomain.BaseDirectory);
//string currentDirectory = Directory.GetCurrentDirectory();
//string filePath = System.IO.Path.Combine(currentDirectory, "Data", "myfile.txt");
// extract the text
//string test = "";
outFileName += #"\test.txt";
// Create a reader for the given PDF file
PdfReader reader = new PdfReader(inFileName);
//outFile = File.CreateText(outFileName);
outFile = new StreamWriter(outFileName, true, System.Text.Encoding.UTF8);
Console.Write("Processing: ");
int totalLen = 68;
float charUnit = ((float)totalLen) / (float)reader.NumberOfPages;
int totalWritten = 0;
float curUnit = 0;
for (int page = 1; page <= reader.NumberOfPages; page++)
{
outFile.Write(ExtractTextFromPDFBytes(reader.GetPageContent(page)) + " ");
// Write the progress.
if (charUnit >= 1.0f)
{
for (int i = 0; i < (int)charUnit; i++)
{
Console.Write("#");
totalWritten++;
}
}
else
{
curUnit += charUnit;
if (curUnit >= 1.0f)
{
for (int i = 0; i < (int)curUnit; i++)
{
Console.Write("#");
totalWritten++;
}
curUnit = 0;
}
}
}
if (totalWritten < totalLen)
{
for (int i = 0; i < (totalLen - totalWritten); i++)
{
Console.Write("#");
}
}
return true;
}
catch(Exception ex)
{
return false;
}
finally
{
if (outFile != null) outFile.Close();
}
}
#endregion
#region ExtractTextFromPDFBytes
/// <summary>
/// This method processes an uncompressed Adobe (text) object
/// and extracts text.
/// </summary>
/// <param name="input">uncompressed</param>
/// <returns></returns>
private string ExtractTextFromPDFBytes(byte[] input)
{
if (input == null || input.Length == 0) return "";
try
{
string resultString = "";
// Flag showing if we are we currently inside a text object
bool inTextObject = false;
// Flag showing if the next character is literal
// e.g. '\\' to get a '\' character or '\(' to get '('
bool nextLiteral = false;
// () Bracket nesting level. Text appears inside ()
int bracketDepth = 0;
// Keep previous chars to get extract numbers etc.:
char[] previousCharacters = new char[_numberOfCharsToKeep];
for (int j = 0; j < _numberOfCharsToKeep; j++) previousCharacters[j] = ' ';
for (int i = 0; i < input.Length; i++)
{
char c = (char)input[i];
if (inTextObject)
{
// Position the text
if (bracketDepth == 0)
{
if (CheckToken(new string[] { "TD", "Td" }, previousCharacters))
{
resultString += "\n\r";
}
else
{
if (CheckToken(new string[] { "'", "T*", "\"" }, previousCharacters))
{
resultString += "\n";
}
else
{
if (CheckToken(new string[] { "Tj" }, previousCharacters))
{
resultString += " ";
}
}
}
}
// End of a text object, also go to a new line.
if (bracketDepth == 0 &&
CheckToken(new string[] { "ET" }, previousCharacters))
{
inTextObject = false;
resultString += " ";
}
else
{
// Start outputting text
if ((c == '(') && (bracketDepth == 0) && (!nextLiteral))
{
bracketDepth = 1;
}
else
{
// Stop outputting text
if ((c == ')') && (bracketDepth == 1) && (!nextLiteral))
{
bracketDepth = 0;
}
else
{
// Just a normal text character:
if (bracketDepth == 1)
{
// Only print out next character no matter what.
// Do not interpret.
if (c == '\\' && !nextLiteral)
{
nextLiteral = true;
}
else
{
if (((c >= ' ') && (c <= '~')) ||
((c >= 128) && (c < 255)))
{
resultString += c.ToString();
}
nextLiteral = false;
}
}
}
}
}
}
// Store the recent characters for
// when we have to go back for a checking
for (int j = 0; j < _numberOfCharsToKeep - 1; j++)
{
previousCharacters[j] = previousCharacters[j + 1];
}
previousCharacters[_numberOfCharsToKeep - 1] = c;
// Start of a text object
if (!inTextObject && CheckToken(new string[] { "BT" }, previousCharacters))
{
inTextObject = true;
}
}
return resultString;
}
catch
{
return "";
}
}
#endregion
#region CheckToken
/// <summary>
/// Check if a certain 2 character token just came along (e.g. BT)
/// </summary>
/// <param name="search">the searched token</param>
/// <param name="recent">the recent character array</param>
/// <returns></returns>
private bool CheckToken(string[] tokens, char[] recent)
{
foreach (string token in tokens)
{
if ((recent[_numberOfCharsToKeep - 3] == token[0]) &&
(recent[_numberOfCharsToKeep - 2] == token[1]) &&
((recent[_numberOfCharsToKeep - 1] == ' ') ||
(recent[_numberOfCharsToKeep - 1] == 0x0d) ||
(recent[_numberOfCharsToKeep - 1] == 0x0a)) &&
((recent[_numberOfCharsToKeep - 4] == ' ') ||
(recent[_numberOfCharsToKeep - 4] == 0x0d) ||
(recent[_numberOfCharsToKeep - 4] == 0x0a))
)
{
return true;
}
}
return false;
}
#endregion
}
}
I'm using this way:
PDFParser pdfParser = new PDFParser();
pdfParser.ExtractText(pdfFile,Path.GetFileNameWithoutExtension(pdfFile) + ".txt");
So the pdf content is written in a txt file.
It works good for certain pdf-s, but for a pdf file that I really need to use, the txt file remains always empty. I didn't get errors, but for some reason it's not writing anything, although as you can see in this screenshot it recognize the pdf,that it has 2 pages...
This is the pdf that I need but the txt always remains empty.(the black lines are added by me, so there are not present when I want to write in the txt)
And this is another pdf. For this the program works ok, and it is written is a txt file. It is much bigger than the other pdf, and still for this I can extract the texts and for the other I can't.
Do you have any idea what can be the problem?

Too long for comment and maybe an answer that you do not like to get:
In PDFs the "Text your see" aka how does a font look and "What the glyphs mean" aka what glyph is mapped to which utf8-letter are separate things.
They are stored in different parts of the pdf - it is utterly possible that a pdf looks totally fine, but if you try to extract text it will give you nothing from it because it only contains the shape of your textglyphs but not theire "meaning".
Try to open the pdf and Select + Copy the text you are after, if you paste that into an editor and noting is there, your pdf lacks the information "what utf8-letter is displayed by this glyph".
OR:
It also might be that your pdf only containts the image of a text - a photo so to say. You can read it, iTextSharp sees only a "picture" - no text.
Those are possible 'why's that would answer your question. As to how to fix it:
There are several questions about corrupt PDFs on SO:
How to repair a PDF file and embed missing fonts
Embedded fonts in PDF: copy and paste problems (this answer)
Copy and Paste relates to text parsing, so the might help you out on how to fix it.
Your edit shows details about your parsing, why don't you leverage iTextSharp for that?
using iTextSharp.text.pdf;
using iTextSharp.text.pdf.parser;
public static string ExtractTextFromPdf(string path)
{
using (PdfReader reader = new PdfReader(path))
{
StringBuilder text = new StringBuilder();
for (int i = 1; i <= reader.NumberOfPages; i++)
{
text.Append(PdfTextExtractor.GetTextFromPage(reader, i));
}
return text.ToString();
}
from: http://www.squarepdf.net/parsing-pdf-files-using-itextsharp
or like here: parse-pdf-with-itextsharp-and-then-extract-specific-text-to-the-screen ?

extracting Arabic text in c# by using itextsharp

I have this code and I'm using it to take the text of a PDF. It's great for a PDF in English but when I'm trying to extract the text in Arabic it shows me something like this.
") + n 9 n <+, + )+ $ # $ +$ F% 9& .< $ : ;"
using (PdfReader reader = new PdfReader(path))
{
ITextExtractionStrategy strategy = new SimpleTextExtractionStrategy();
String text = "";
for (int i = 1; i <= reader.NumberOfPages; i++)
{
text = PdfTextExtractor.GetTextFromPage(reader, i,strategy);
}
}

I had to change the strategy like this
var t = PdfTextExtractor.GetTextFromPage(reader, i, new LocationTextExtractionStrategy());
var te = Convert(t);
and this function to reverse the Arabic words and keep the English
private string Convert(string source)
{
string arabicWord = string.Empty;
StringBuilder sbDestination = new StringBuilder();
foreach (var ch in source)
{
if (IsArabic(ch))
arabicWord += ch;
else
{
if (arabicWord != string.Empty)
sbDestination.Append(Reverse(arabicWord));
sbDestination.Append(ch);
arabicWord = string.Empty;
}
}
// if the last word was arabic
if (arabicWord != string.Empty)
sbDestination.Append(Reverse(arabicWord));
return sbDestination.ToString();
}
private bool IsArabic(char character)
{
if (character >= 0x600 && character <= 0x6ff)
return true;
if (character >= 0x750 && character <= 0x77f)
return true;
if (character >= 0xfb50 && character <= 0xfc3f)
return true;
if (character >= 0xfe70 && character <= 0xfefc)
return true;
return false;
}
// Reverse the characters of string
string Reverse(string source)
{
return new string(source.ToCharArray().Reverse().ToArray());
}

Application that indents an unindented code in C#

My application should read a C# code sample that is unindented, then indent the code programatically. The way I am doing it may not be correct but still could achieve partial results.
I could set white spaces when a { is found then continue with the same amount of space for rest of the lines being read. When another { is found again add spaces and continue with this new space for rest of lines. For that this is what I did:
private void btn_format_Click(object sender, EventArgs e)
{
string lineInfo = "";
string fl = "";
string ctab= char.ConvertFromUtf32(32)+char.ConvertFromUtf32(32)+char.ConvertFromUtf32(32);
foreach (string line in txt_codepage.Lines) // text_codepage is a textbox with code
{
if (line.Contains("{"))
{
string l = line.Replace("{", ctab+"{");
lineInfo = lineInfo + (l + "\n");
fl = fl + ctab;
ctab = ctab + ctab;
}
else
{
lineInfo = lineInfo + (char.ConvertFromUtf32(32)+fl+ line + "\n");
}
I could achieve the proper indentation that I want till here. Now when I find a } I should do the reverse process but unfortunately that is not possible with strings. The reverse process that I meant is this:
if (line.Contains("}"))
{
string l = line.Replace(ctab + "}", "}");
lineInfo = lineInfo + (l + "\n");
fl = fl - ctab;
ctab = ctab - ctab;
}
else
{
lineInfo = lineInfo - (char.ConvertFromUtf32(32) + fl + line + "\n");
}
}
MessageBox.Show(lineInfo.ToString());
I know the above part of the code is a complete blunder but let me know how to achieve it in correct way

If you want parse string, you should use StringBuilder instead string concatenations (concatenations is to slow). I wrote some code, to demonstrate how you can parse CS or other code. It is not a full example, just a basic concepts.
If you want learn more about parsers you can read Compilers: Principles, Techniques, and Tools.
public static string IndentCSharpCode(string code)
{
const string INDENT_STEP = " ";
if (string.IsNullOrWhiteSpace(code))
{
return code;
}
var result = new StringBuilder();
var indent = string.Empty;
var lineContent = false;
var stringDefinition = false;
for (var i = 0; i < code.Length; i++)
{
var ch = code[i];
if (ch == '"' && !stringDefinition)
{
result.Append(ch);
stringDefinition = true;
continue;
}
if (ch == '"' && stringDefinition)
{
result.Append(ch);
stringDefinition = false;
continue;
}
if (stringDefinition)
{
result.Append(ch);
continue;
}
if (ch == '{' && !stringDefinition)
{
if (lineContent)
{
result.AppendLine();
}
result.Append(indent).Append("{");
if (lineContent)
{
result.AppendLine();
}
indent += INDENT_STEP;
lineContent = false;
continue;
}
if (ch == '}' && !stringDefinition)
{
if (indent.Length != 0)
{
indent = indent.Substring(0, indent.Length - INDENT_STEP.Length);
}
if (lineContent)
{
result.AppendLine();
}
result.Append(indent).Append("}");
if (lineContent)
{
result.AppendLine();
}
lineContent = false;
continue;
}
if (ch == '\r')
{
continue;
}
if ((ch == ' ' || ch == '\t') && !lineContent)
{
continue;
}
if (ch == '\n')
{
lineContent = false;
result.AppendLine();
continue;
}
if (!lineContent)
{
result.Append(indent);
lineContent = true;
}
result.Append(ch);
}
return result.ToString();
}

You can go and check out codemaid, an open source VS add in for cleaning code

Remove all of the whitespace from the line using String.Trim() and then add just the tabs you want. Also, your code would be much more readable if you could avoid char.ConvertFromUtf32(32) - why write that instead of " " or ' '?

c# Registry to XML Invalid character issue

I have a problem when trying to create an XML file from registry. On my laptop(W7 64b) it is working fine, the xml file is generated but on another computer (Xp 32b) an exception is thrown : System.ArgumentException '.', hexadecimal values 0x00, is an invalid character. I have read few useful things about it but I don't know how to solve in this case, here is the code :
try
{
string regPath = "SOFTWARE\\IPS";
XElement xRegRoot = new XElement("Root", new XAttribute("Registry", regPath));
ReadRegistry(regPath, xRegRoot);
string xmlStringReg = xRegRoot.ToString();
XmlDocument docR = new XmlDocument();
docR.LoadXml(xmlStringReg);
docR.Save(AppDomain.CurrentDomain.BaseDirectory + "\\_RegistryList.xml");
}
catch (System.Exception ex)
{
Console.WriteLine(ex.ToString());
LogToFile(ex.ToString());
}
private static void ReadRegistry(string keyPath, XElement xRegRoot)
{
string[] subKeys=null;
RegistryKey HKLM = Registry.LocalMachine;
RegistryKey RegKey = HKLM.OpenSubKey(keyPath);
try
{
subKeys = RegKey.GetSubKeyNames();
foreach (string subKey in subKeys)
{
string fullPath = keyPath + "\\" + subKey;
Console.WriteLine("\r\nKey Name | " + fullPath);
LogToFile("Key Name | " + fullPath);
XElement xregkey = new XElement("RegKeyName", new XAttribute("FullName", fullPath), new XAttribute("Name", subKey));
xRegRoot.Add(xregkey);
ReadRegistry(fullPath, xRegRoot);
}
string[] subVals = RegKey.GetValueNames();
foreach (string val in subVals)
{
string keyName = val;
string keyType = RegKey.GetValueKind(val).ToString();
string keyValue = RegKey.GetValue(val).ToString();
Console.WriteLine("Key Value | " + keyType + " | " + keyName + " | " + keyValue);
LogToFile("Key " + keyType + " | " + keyName + " | " + keyValue);
XElement xregvalue = new XElement("RegKeyValue", new XAttribute("keyType", keyType), new XAttribute("keyName", keyName), new XAttribute("keyValue", keyValue));
xRegRoot.Add(xregvalue);
}
}
catch (System.Exception ex)
{
Console.WriteLine(ex.ToString());
LogToFile(ex.ToString());
}
}
Thanks in advance.

I did some experiments:
new XElement("foo\x00bar") throws on construction.
new XAttribute("foo\x00bar", "baz") throws on construction.
new XText("foo\x00bar") throws only when calling .ToString().
new XAttribute("foo", "bar\x00baz") is equivalent to new XAttribute("foo", new XText("bar\x00baz")), so it won't throw on construction.
I did not manage to make any of the registry-methods return a string with null-characters, but you should be able to find where this is returned yourself.

You can read more about it here: http://seattlesoftware.wordpress.com/2008/09/11/hexadecimal-value-0-is-an-invalid-character/
And more about it here: XElement & UTF-8 Issue
A valid list of xml chars are here
http://en.wikipedia.org/wiki/Valid_characters_in_XML
But essentially you can fix it by removing illegal chars before serialising
/// <summary>
/// Remove illegal XML characters from a string.
/// </summary>
public string SanitizeXmlString(string xml)
{
if (string.IsNullOrEmpty(value))
{
return value;
}
StringBuilder buffer = new StringBuilder(xml.Length);
foreach (char c in xml)
{
if (IsLegalXmlChar(c))
{
buffer.Append(c);
}
}
return buffer.ToString();
}
/// <summary>
/// Whether a given character is allowed by XML 1.0.
/// </summary>
public bool IsLegalXmlChar(int character)
{
return
(
character == 0x9 /* == '\t' == 9 */ ||
character == 0xA /* == '\n' == 10 */ ||
character == 0xD /* == '\r' == 13 */ ||
(character >= 0x20 && character <= 0xD7FF ) ||
(character >= 0xE000 && character <= 0xFFFD ) ||
(character >= 0x10000 && character <= 0x10FFFF)
);
}

Here are a couple little improvements that a) compile, and b) handle surrogate pairs:
/// <summary>
/// Remove illegal XML characters from a string.
/// </summary>
public static string SanitizeString(string s)
{
if (string.IsNullOrEmpty(s))
{
return s;
}
StringBuilder buffer = new StringBuilder(s.Length);
for (int i = 0; i < s.Length; i++)
{
int code;
try
{
code = Char.ConvertToUtf32(s, i);
}
catch (ArgumentException)
{
continue;
}
if (IsLegalXmlChar(code))
buffer.Append(Char.ConvertFromUtf32(code));
if (Char.IsSurrogatePair(s, i))
i++;
}
return buffer.ToString();
}
/// <summary>
/// Whether a given character is allowed by XML 1.0.
/// </summary>
private static bool IsLegalXmlChar(int codePoint)
{
return (codePoint == 0x9 ||
codePoint == 0xA ||
codePoint == 0xD ||
(codePoint >= 0x20 && codePoint <= 0xD7FF) ||
(codePoint >= 0xE000 && codePoint <= 0xFFFD) ||
(codePoint >= 0x10000/* && character <= 0x10FFFF*/) //it's impossible to get a code point bigger than 0x10FFFF because Char.ConvertToUtf32 would have thrown an exception
);
}

Unable to find csv file on ASP.NET

Anyone who knows how to solve this error please help me. Below is my code for reading the csv file. When I tried to upload it showed me *Server Error in '/' Application. Could not find file 'C:/...csv * Im a beginner in c#.
ReadCSV
string filename = FileUpload1.PostedFile.FileName;
using (CsvFileReader reader = new CsvFileReader(filename))
{
CsvRow row = new CsvRow();
while (reader.ReadRow(row))
{
foreach (string s in row)
{
Console.Write(s);
Console.Write(" ");
TextBox1.Text += s;
}
Console.WriteLine();
}
}
CSVClass
public class CsvFileReader : StreamReader
{
public CsvFileReader(Stream stream)
: base(stream)
{
}
public CsvFileReader(string filename): base(filename)
{
}
/// <summary>
/// Reads a row of data from a CSV file
/// </summary>
/// <param name="row"></param>
/// <returns></returns>
public bool ReadRow(CsvRow row)
{
row.LineText = ReadLine();
if (String.IsNullOrEmpty(row.LineText))
return false;
int pos = 0;
int rows = 0;
while (pos < row.LineText.Length)
{
string value;
// Special handling for quoted field
if (row.LineText[pos] == '"')
{
// Skip initial quote
pos++;
// Parse quoted value
int start = pos;
while (pos < row.LineText.Length)
{
// Test for quote character
if (row.LineText[pos] == '"')
{
// Found one
pos++;
// If two quotes together, keep one
// Otherwise, indicates end of value
if (pos >= row.LineText.Length || row.LineText[pos] != '"')
{
pos--;
break;
}
}
pos++;
}
value = row.LineText.Substring(start, pos - start);
value = value.Replace("\"\"", "\"");
}
else
{
// Parse unquoted value
int start = pos;
while (pos < row.LineText.Length && row.LineText[pos] != ',')
pos++;
value = row.LineText.Substring(start, pos - start);
}
// Add field to list
if (rows < row.Count)
row[rows] = value;
else
row.Add(value);
rows++;
// Eat up to and including next comma
while (pos < row.LineText.Length && row.LineText[pos] != ',')
pos++;
if (pos < row.LineText.Length)
pos++;
}
// Delete any unused items
while (row.Count > rows)
row.RemoveAt(rows);
// Return true if any columns read
return (row.Count > 0);
}
}

FileUpload1.PostedFile.FileName is the filename from your client/browser - it does not contain a path...
You either use FileUpload1.PostedFile.InputStream to access it
using (CsvFileReader reader = new CsvFileReader(FileUpload1.PostedFile.InputStream))
OR you first save it to disk (anywhere you have needed permissions) via FileUpload1.PostedFile.SaveAs and then access that file.

You need to save the file that is being uploaded to disk first. Something like this:
string fileSavePath= Sever.MapPath("/files/" + FileUpload1.PostedFile.FileName);
FileUpload1.SaveAs(fileSavePath);
using (CsvFileReader reader = new CsvFileReader(fileSavePath))
....
Haven't tested this code, but should give you a starting point.