Related
A few days ago i switched from tensorflow to fastai for my c# Project. But now i am facing a problem with my normalisation. For both i use an onnx pipeline to load the model and the data.
var onnxPipeline = mLContext.Transforms.ResizeImages(resizing: ImageResizingEstimator.ResizingKind.Fill, outputColumnName: inputName,
imageWidth: ImageSettings.imageWidth, imageHeight: ImageSettings.imageHeight,
inputColumnName: nameof(ImageInputData.Image))
.Append(mLContext.Transforms.ExtractPixels(outputColumnName: inputName, interleavePixelColors: true, scaleImage: 1 / 255f))
.Append(mLContext.Transforms.ApplyOnnxModel(outputColumnName: outputName, inputColumnName: inputName, modelFile: onnxModelPath));
var emptyData = mLContext.Data.LoadFromEnumerable(new List<ImageInputData>());
var onnxModel = onnxPipeline.Fit(emptyData);
with
class ImageInputData
{
[ImageType(ImageSettings.imageHeight, ImageSettings.imageWidth)]
public Bitmap Image { get; set; }
public ImageInputData(byte[] image)
{
using (var ms = new MemoryStream(image))
{
Image = new Bitmap(ms);
}
}
public ImageInputData(Bitmap image)
{
Image = image;
}
}
After using fastai i learned, that the models get better accuracy if the data is normalized with a specific mean and standard deviation (because i used the resnet34 model it should be means { 0.485, 0.456, 0.406 } stds = { 0.229, 0.224, 0.225 } respectively).
So the pixelvalues (for each color ofc.) have to be transformed with those values to match the trainings images. But how can i achive this in C#?
What i tried so far is:
int imageSize = 256;
double[] means = new double[] { 0.485, 0.456, 0.406 }; // used in fastai model
double[] stds = new double[] { 0.229, 0.224, 0.225 };
Bitmap bitmapImage = inputBitmap;
Image image = bitmapImage;
Color[] pixels = new Color[imageSize * imageSize];
for (int x = 0; x < bitmapImage.Width; x++)
{
for (int y = 0; y < bitmapImage.Height; y++)
{
Color pixel = bitmapImage.GetPixel(x, y);
pixels[x + y] = pixel;
double red = (pixel.R - (means[0] * 255)) / (stds[0] * 255); // *255 to scale the mean and std values to the Bitmap
double gre = (pixel.G - (means[1] * 255)) / (stds[1] * 255);
double blu = (pixel.B - (means[2] * 255)) / (stds[2] * 255);
Color pixel_n = Color.FromArgb(pixel.A, (int)red, (int)gre, (int)blu);
bitmapImage.SetPixel(x, y, pixel_n);
}
}
Ofcourse its not working, because the Colorvalues can`t be negative (which i realised only later).
But how can i achive this normalisation between -1 and 1 for my model in C# with the onnx-model?
Is there a different way to feed the model or to handle the normalisation?
Any help would be appreciated!
One way to solve this problem is to switch from an onnx pipeline to an onnx Inferencesession, which is in my view simpler and better to understand:
public List<double> UseOnnxSession(Bitmap image, string onnxModelPath)
{
double[] means = new double[] { 0.485, 0.456, 0.406 };
double[] stds = new double[] { 0.229, 0.224, 0.225 };
using (var session = new InferenceSession(onnxModelPath))
{
List<double> scores = new List<double>();
Tensor<float> t1 = ConvertImageToFloatData(image, means, stds);
List<float> fl = new List<float>();
var inputMeta = session.InputMetadata;
var inputs = new List<NamedOnnxValue>()
{
NamedOnnxValue.CreateFromTensor<float>("input_1", t1)
};
using (var results = session.Run(inputs))
{
foreach (var r in results)
{
var x = r.AsTensor<float>().First();
var y = r.AsTensor<float>().Last();
var softmaxScore = Softmax(new double[] { x, y });
scores.Add(softmaxScore[0]);
scores.Add(softmaxScore[1]);
}
}
return scores;
}
}
// Create your Tensor and add transformations as you need.
public static Tensor<float> ConvertImageToFloatData(Bitmap image, double[] means, double[] std)
{
Tensor<float> data = new DenseTensor<float>(new[] { 1, 3, image.Width, image.Height });
for (int x = 0; x < image.Width; x++)
{
for (int y = 0; y < image.Height; y++)
{
Color color = image.GetPixel(x, y);
var red = (color.R - (float)means[0] * 255) / ((float)std[0] * 255);
var gre = (color.G - (float)means[1] * 255) / ((float)std[1] * 255);
var blu = (color.B - (float)means[2] * 255) / ((float)std[2] * 255);
data[0, 0, x, y] = red;
data[0, 1, x, y] = gre;
data[0, 2, x, y] = blu;
}
}
return data;
}
Also i have to use my own Softmax method on these scores to get the real probabilities out of my model:
public double[] Softmax(double[] values)
{
double[] ret = new double[values.Length];
double maxExp = values.Select(Math.Exp).Sum();
for (int i = 0; i < values.Length; i++)
{
ret[i] = Math.Round((Math.Exp(values[i]) / maxExp), 4);
}
return ret;
}
Hope this helps someone who has a similar Problem.
I am trying to implement the unsharp masking method on emgucv using c#.
The python code I have now is (ref):
def unsharp_mask(image, kernel_size=(5, 5), sigma=1.0, amount=1.0, threshold=0):
"""Return a sharpened version of the image, using an unsharp mask."""
# For details on unsharp masking, see:
# https://en.wikipedia.org/wiki/Unsharp_masking
# https://homepages.inf.ed.ac.uk/rbf/HIPR2/unsharp.htm
blurred = cv.GaussianBlur(image, kernel_size, sigma)
sharpened = float(amount + 1) * image - float(amount) * blurred
sharpened = np.maximum(sharpened, np.zeros(sharpened.shape))
sharpened = np.minimum(sharpened, 255 * np.ones(sharpened.shape))
sharpened = sharpened.round().astype(np.uint8)
if threshold > 0:
low_contrast_mask = np.absolute(image - blurred) < threshold
np.copyto(sharpened, image, where=low_contrast_mask)
return sharpened
The c# code I have now cannot do the work as the above code does. Does anyone know how to implement it emgu cv using c#?
public static void GetMat(Image<Gray, byte> srcimg, Image<Gray, byte> imgBlurred, ref Mat dst, int nAmount = 200)
{
float amount = nAmount / 100f;
using (Image<Gray, byte> dst_temp = new Image<Gray, byte>(srcimg.Width, srcimg.Height))
{
for (int v = 0; v < srcimg.Height; v++)
{
for (int u = 0; u < srcimg.Width; u++)
{
byte a = srcimg.Data[v, u, 0]; //Get Pixel Color | fast way
byte b = imgBlurred.Data[v, u, 0];
int c = (int)(a * (1 + amount) - (amount * b));
if (c < 0) c = 0;
if (c > 255) c = 255;
dst_temp.Data[v, u, 0] = (byte)c;
}
}
dst = dst_temp.Mat.Clone();
}
}
public static void getSharpenImage(Mat src, ref Mat dst, int nAmount = 200, double sigma = 3, int threshold = 0)
{
float amount = nAmount / 100.0F;
using (Mat imgBlurred = new Mat())
{
CvInvoke.GaussianBlur(src, imgBlurred, new System.Drawing.Size(0, 0), sigma, sigma);
using (Mat mask_temp = new Mat())
{
CvInvoke.AbsDiff(src, imgBlurred, mask_temp);
using (Mat lowcontrastmask = new Mat())
{
CvInvoke.Threshold(mask_temp, lowcontrastmask, threshold, 255, ThresholdType.BinaryInv);
GetMat(src.ToImage<Gray, byte>(), imgBlurred.ToImage<Gray, byte>(), ref dst);
src.CopyTo(dst, lowcontrastmask);
}
}
}
}
https://www.idtools.com.au/unsharp-masking-python-opencv/ has a python solution.
the following works in C#:
Mat blurredImage = new Mat();
Mat lapImage = new Mat();
CvInvoke.MedianBlur(grayImage, blurredImage, 1);
CvInvoke.Laplacian(blurredImage, lapImage, blurredImage.Depth);
blurredImage -= (0.9*lapImage);
I am trying to automate something with my C# application, for which I use a bitmap detection system to detect if an icon has appeared on screen. This works perfectly on a PC. However, when I put the application on a server, it never works. I am using a Google Cloud instance with a Tesla K80, 2 vcpus running Windows server 2012.
Here is my code:
// Capture the current screen as a bitmap
public static Bitmap CaptureScreen()
{
// Bitmap format
Bitmap ScreenCapture = new Bitmap(Screen.PrimaryScreen.Bounds.Width, Screen.PrimaryScreen.Bounds.Height);
// Capture screen
Graphics GFX = Graphics.FromImage(ScreenCapture);
GFX.CopyFromScreen(Screen.PrimaryScreen.Bounds.X,
Screen.PrimaryScreen.Bounds.Y, 0, 0,
ScreenCapture.Size, CopyPixelOperation.SourceCopy);
return ScreenCapture;
}
// Find a list of all the points of a bitmap within another bitmap
public static List<Point> FindBitmapsEntry(Bitmap SourceBitmap, Bitmap SearchedBitmap)
{
#region Arguments check
if (SourceBitmap == null || SearchedBitmap == null)
throw new ArgumentNullException();
if (SourceBitmap.PixelFormat != SearchedBitmap.PixelFormat)
throw new ArgumentException("Pixel formats aren't equal.");
if (SourceBitmap.Width < SearchedBitmap.Width || SourceBitmap.Height < SearchedBitmap.Height)
throw new ArgumentException("Size of SearchedBitmap is bigger than SourceBitmap!");
#endregion
var PixelFormatSize = Image.GetPixelFormatSize(SourceBitmap.PixelFormat) / 8;
// Copy SourceBitmap to byte array
var SourceBitmapData = SourceBitmap.LockBits(new Rectangle(0, 0, SourceBitmap.Width, SourceBitmap.Height),
ImageLockMode.ReadOnly, SourceBitmap.PixelFormat);
var SourceBitmapByteLength = SourceBitmapData.Stride * SourceBitmap.Height;
var SourceBytes = new byte[SourceBitmapByteLength];
Marshal.Copy(SourceBitmapData.Scan0, SourceBytes, 0, SourceBitmapByteLength);
SourceBitmap.UnlockBits(SourceBitmapData);
// Copy SearchedBitmap to byte array
var SearchingBitmapData =
SearchedBitmap.LockBits(new Rectangle(0, 0, SearchedBitmap.Width, SearchedBitmap.Height),
ImageLockMode.ReadOnly, SearchedBitmap.PixelFormat);
var SearchingBitmapByteLength = SearchingBitmapData.Stride * SearchedBitmap.Height;
var SearchingBytes = new byte[SearchingBitmapByteLength];
Marshal.Copy(SearchingBitmapData.Scan0, SearchingBytes, 0, SearchingBitmapByteLength);
SearchedBitmap.UnlockBits(SearchingBitmapData);
var PointsList = new List<Point>();
// Searching entries, minimizing searching zones
// SourceBitmap.Height - SearchedBitmap.Height + 1
for (var MainY = 0; MainY < SourceBitmap.Height - SearchedBitmap.Height + 1; MainY++)
{
var SourceY = MainY * SourceBitmapData.Stride;
for (var MainX = 0; MainX < SourceBitmap.Width - SearchedBitmap.Width + 1; MainX++)
{
// MainY & MainX - pixel coordinates of SourceBitmap
// SourceY + SourceX = pointer in array SourceBitmap bytes
var SourceX = MainX * PixelFormatSize;
var IsEqual = true;
for (var c = 0; c < PixelFormatSize; c++)
{
// Check through the bytes in pixel
if (SourceBytes[SourceX + SourceY + c] == SearchingBytes[c])
continue;
IsEqual = false;
break;
}
if (!IsEqual) continue;
var ShouldStop = false;
// Find first equation and search deeper
for (var SecY = 0; SecY < SearchedBitmap.Height; SecY++)
{
var SearchY = SecY * SearchingBitmapData.Stride;
var SourceSecY = (MainY + SecY) * SourceBitmapData.Stride;
for (var SecX = 0; SecX < SearchedBitmap.Width; SecX++)
{
// SecX & SecY - coordinates of SearchingBitmap
// SearchX + SearchY = pointer in array SearchingBitmap bytes
var SearchX = SecX * PixelFormatSize;
var SourceSecX = (MainX + SecX) * PixelFormatSize;
for (var c = 0; c < PixelFormatSize; c++)
{
// Check through the bytes in pixel
if (SourceBytes[SourceSecX + SourceSecY + c] == SearchingBytes[SearchX + SearchY + c]) continue;
// Not equal - abort iteration
ShouldStop = true;
break;
}
if (ShouldStop) break;
}
if (ShouldStop) break;
}
if (!ShouldStop) // Bitmap is found
{
PointsList.Add(new Point(MainX, MainY));
}
}
}
return PointsList;
}
And here is how I use it:
Bitmap HighlightBitmap = new Bitmap(Resources.icon);
Bitmap CurrentScreen = CaptureScreen();
List<Point> HighlightPoints = FindBitmapsEntry(CurrentScreen, HighlightBitmap);
with this HighlightPoints[0] is supposed to give me the first point the two bitmaps (icon, screenshot) collide. But as mentioned before, it just doesn't work on the server.
Thanks in advance!
P.S. I am using the server with a RDP so it does have a visual interface to work with
In my previous question, I transformed this image:
into this:
which Tesseract OCR interprets as this:
1O351
Putting a frame around the image
actually improves the OCR result.
1CB51
However, I need all 5 characters to OCR correctly, so as an experiment I used Paint.NET to rotate and align each individual letter into its proper orientation:
Resulting in the correct answer:
1CB52
How would I go about performing this correction in C#?
I've done a bit of research on various text alignment algorithms, but they all assume the existence of lines of text in the source image, lines from which you can derive a rotation angle, but which already contain the proper spacing and orientation relationships between the letters.
You can use the code in the following code project article to segment each individual character. However, when trying to deskew these characters individually any result you get is not going to be very good because there isn't very much information to go off of.
I tried using AForge.NETs HoughLineTransformation class and I got angles in the range of 80 - 90 degrees. So I tried using the following code to deskew them:
private static Bitmap DeskewImageByIndividualChars(Bitmap targetBitmap)
{
IDictionary<Rectangle, Bitmap> characters = new CCL().Process(targetBitmap);
using (Graphics g = Graphics.FromImage(targetBitmap))
{
foreach (var character in characters)
{
double angle;
BitmapData bitmapData = character.Value.LockBits(new Rectangle(Point.Empty, character.Value.Size), ImageLockMode.ReadWrite, PixelFormat.Format8bppIndexed);
try
{
HoughLineTransformation hlt = new HoughLineTransformation();
hlt.ProcessImage(bitmapData);
angle = hlt.GetLinesByRelativeIntensity(0.5).Average(l => l.Theta);
}
finally
{
character.Value.UnlockBits(bitmapData);
}
using (Bitmap bitmap = RotateImage(character.Value, 90 - angle, Color.White))
{
g.DrawImage(bitmap, character.Key.Location);
}
}
}
return targetBitmap;
}
With the RotateImage method taken from here. However, the results didn't seem to be the best. Maybe you can try and make them better.
Here is the code from the code project article for your reference. I have made a few changes to it so that it behaves a bit safer, such as adding try-finally around the LockBits and disposing of objects properly using the using statement etc.
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
namespace ConnectedComponentLabeling
{
public class CCL
{
private Bitmap _input;
private int[,] _board;
public IDictionary<Rectangle, Bitmap> Process(Bitmap input)
{
_input = input;
_board = new int[_input.Width, _input.Height];
Dictionary<int, List<Pixel>> patterns = Find();
var images = new Dictionary<Rectangle, Bitmap>();
foreach (KeyValuePair<int, List<Pixel>> pattern in patterns)
{
using (Bitmap bmp = CreateBitmap(pattern.Value))
{
images.Add(GetBounds(pattern.Value), (Bitmap)bmp.Clone());
}
}
return images;
}
protected virtual bool CheckIsBackGround(Pixel currentPixel)
{
return currentPixel.color.A == 255 && currentPixel.color.R == 255 && currentPixel.color.G == 255 && currentPixel.color.B == 255;
}
private unsafe Dictionary<int, List<Pixel>> Find()
{
int labelCount = 1;
var allLabels = new Dictionary<int, Label>();
BitmapData imageData = _input.LockBits(new Rectangle(0, 0, _input.Width, _input.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
try
{
int bytesPerPixel = 3;
byte* scan0 = (byte*)imageData.Scan0.ToPointer();
int stride = imageData.Stride;
for (int i = 0; i < _input.Height; i++)
{
byte* row = scan0 + (i * stride);
for (int j = 0; j < _input.Width; j++)
{
int bIndex = j * bytesPerPixel;
int gIndex = bIndex + 1;
int rIndex = bIndex + 2;
byte pixelR = row[rIndex];
byte pixelG = row[gIndex];
byte pixelB = row[bIndex];
Pixel currentPixel = new Pixel(new Point(j, i), Color.FromArgb(pixelR, pixelG, pixelB));
if (CheckIsBackGround(currentPixel))
{
continue;
}
IEnumerable<int> neighboringLabels = GetNeighboringLabels(currentPixel);
int currentLabel;
if (!neighboringLabels.Any())
{
currentLabel = labelCount;
allLabels.Add(currentLabel, new Label(currentLabel));
labelCount++;
}
else
{
currentLabel = neighboringLabels.Min(n => allLabels[n].GetRoot().Name);
Label root = allLabels[currentLabel].GetRoot();
foreach (var neighbor in neighboringLabels)
{
if (root.Name != allLabels[neighbor].GetRoot().Name)
{
allLabels[neighbor].Join(allLabels[currentLabel]);
}
}
}
_board[j, i] = currentLabel;
}
}
}
finally
{
_input.UnlockBits(imageData);
}
Dictionary<int, List<Pixel>> patterns = AggregatePatterns(allLabels);
patterns = RemoveIntrusions(patterns, _input.Width, _input.Height);
return patterns;
}
private Dictionary<int, List<Pixel>> RemoveIntrusions(Dictionary<int, List<Pixel>> patterns, int width, int height)
{
var patternsCleaned = new Dictionary<int, List<Pixel>>();
foreach (var pattern in patterns)
{
bool bad = false;
foreach (Pixel item in pattern.Value)
{
//Horiz
if (item.Position.X == 0)
bad = true;
else if (item.Position.Y == width - 1)
bad = true;
//Vert
else if (item.Position.Y == 0)
bad = true;
else if (item.Position.Y == height - 1)
bad = true;
}
if (!bad)
patternsCleaned.Add(pattern.Key, pattern.Value);
}
return patternsCleaned;
}
private IEnumerable<int> GetNeighboringLabels(Pixel pix)
{
var neighboringLabels = new List<int>();
for (int i = pix.Position.Y - 1; i <= pix.Position.Y + 2 && i < _input.Height - 1; i++)
{
for (int j = pix.Position.X - 1; j <= pix.Position.X + 2 && j < _input.Width - 1; j++)
{
if (i > -1 && j > -1 && _board[j, i] != 0)
{
neighboringLabels.Add(_board[j, i]);
}
}
}
return neighboringLabels;
}
private Dictionary<int, List<Pixel>> AggregatePatterns(Dictionary<int, Label> allLabels)
{
var patterns = new Dictionary<int, List<Pixel>>();
for (int i = 0; i < _input.Height; i++)
{
for (int j = 0; j < _input.Width; j++)
{
int patternNumber = _board[j, i];
if (patternNumber != 0)
{
patternNumber = allLabels[patternNumber].GetRoot().Name;
if (!patterns.ContainsKey(patternNumber))
{
patterns[patternNumber] = new List<Pixel>();
}
patterns[patternNumber].Add(new Pixel(new Point(j, i), Color.Black));
}
}
}
return patterns;
}
private unsafe Bitmap CreateBitmap(List<Pixel> pattern)
{
int minX = pattern.Min(p => p.Position.X);
int maxX = pattern.Max(p => p.Position.X);
int minY = pattern.Min(p => p.Position.Y);
int maxY = pattern.Max(p => p.Position.Y);
int width = maxX + 1 - minX;
int height = maxY + 1 - minY;
Bitmap bmp = DrawFilledRectangle(width, height);
BitmapData imageData = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadWrite, PixelFormat.Format24bppRgb);
try
{
byte* scan0 = (byte*)imageData.Scan0.ToPointer();
int stride = imageData.Stride;
foreach (Pixel pix in pattern)
{
scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride] = pix.color.B;
scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride + 1] = pix.color.G;
scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride + 2] = pix.color.R;
}
}
finally
{
bmp.UnlockBits(imageData);
}
return bmp;
}
private Bitmap DrawFilledRectangle(int x, int y)
{
Bitmap bmp = new Bitmap(x, y);
using (Graphics graph = Graphics.FromImage(bmp))
{
Rectangle ImageSize = new Rectangle(0, 0, x, y);
graph.FillRectangle(Brushes.White, ImageSize);
}
return bmp;
}
private Rectangle GetBounds(List<Pixel> pattern)
{
var points = pattern.Select(x => x.Position);
var x_query = points.Select(p => p.X);
int xmin = x_query.Min();
int xmax = x_query.Max();
var y_query = points.Select(p => p.Y);
int ymin = y_query.Min();
int ymax = y_query.Max();
return new Rectangle(xmin, ymin, xmax - xmin, ymax - ymin);
}
}
}
With the above code I got the following input/output:
As you can see the B has rotated quite well but the others aren't as good.
An alternative to trying to deskew the individual characters is to find there location using the segmentation routine above. Then passing each individual character through to your recognition engine separately and seeing if this improves your results.
I have used the following method to find the angle of the character using the List<Pixel> from inside the CCL class. It works by finding the angle between the "bottom left" and "bottom right" points. I haven't tested if it works if the character is rotated the other way around.
private double GetAngle(List<Pixel> pattern)
{
var pixels = pattern.Select(p => p.Position).ToArray();
Point bottomLeft = pixels.OrderByDescending(p => p.Y).ThenBy(p => p.X).First();
Point rightBottom = pixels.OrderByDescending(p => p.X).ThenByDescending(p => p.Y).First();
int xDiff = rightBottom.X - bottomLeft.X;
int yDiff = rightBottom.Y - bottomLeft.Y;
double angle = Math.Atan2(yDiff, xDiff) * 180 / Math.PI;
return -angle;
}
Note my drawing code is a bit broken so that is why the 5 is cut off on the right but this code produces the following output:
Note that the B and the 5 are rotated further than you'd expect because of their curvature.
Using the following code by getting the angle from the left and right edges and then choosing the best one, the rotations seems to be better. Note I have only tested it with letters that need rotating clockwise so if they need to go the opposite way it might not work too well.
This also "quadrants" the pixels so that each pixel is chosen from it's own quadrant as not to get two that are too nearby.
The idea in selecting the best angle is if they are similar, at the moment within 1.5 degrees of each other but can easily be updated, average them. Else we pick the one that is closest to zero.
private double GetAngle(List<Pixel> pattern, Rectangle bounds)
{
int halfWidth = bounds.X + (bounds.Width / 2);
int halfHeight = bounds.Y + (bounds.Height / 2);
double leftEdgeAngle = GetAngleLeftEdge(pattern, halfWidth, halfHeight);
double rightEdgeAngle = GetAngleRightEdge(pattern, halfWidth, halfHeight);
if (Math.Abs(leftEdgeAngle - rightEdgeAngle) <= 1.5)
{
return (leftEdgeAngle + rightEdgeAngle) / 2d;
}
if (Math.Abs(leftEdgeAngle) > Math.Abs(rightEdgeAngle))
{
return rightEdgeAngle;
}
else
{
return leftEdgeAngle;
}
}
private double GetAngleLeftEdge(List<Pixel> pattern, double halfWidth, double halfHeight)
{
var topLeftPixels = pattern.Select(p => p.Position).Where(p => p.Y < halfHeight && p.X < halfWidth).ToArray();
var bottomLeftPixels = pattern.Select(p => p.Position).Where(p => p.Y > halfHeight && p.X < halfWidth).ToArray();
Point topLeft = topLeftPixels.OrderBy(p => p.X).ThenBy(p => p.Y).First();
Point bottomLeft = bottomLeftPixels.OrderByDescending(p => p.Y).ThenBy(p => p.X).First();
int xDiff = bottomLeft.X - topLeft.X;
int yDiff = bottomLeft.Y - topLeft.Y;
double angle = Math.Atan2(yDiff, xDiff) * 180 / Math.PI;
return 90 - angle;
}
private double GetAngleRightEdge(List<Pixel> pattern, double halfWidth, double halfHeight)
{
var topRightPixels = pattern.Select(p => p.Position).Where(p => p.Y < halfHeight && p.X > halfWidth).ToArray();
var bottomRightPixels = pattern.Select(p => p.Position).Where(p => p.Y > halfHeight && p.X > halfWidth).ToArray();
Point topRight = topRightPixels.OrderBy(p => p.Y).ThenByDescending(p => p.X).First();
Point bottomRight = bottomRightPixels.OrderByDescending(p => p.X).ThenByDescending(p => p.Y).First();
int xDiff = bottomRight.X - topRight.X;
int yDiff = bottomRight.Y - topRight.Y;
double angle = Math.Atan2(xDiff, yDiff) * 180 / Math.PI;
return Math.Abs(angle);
}
This now produces the following output, again my drawing code is slightly broken. Note that the C looks to not have deskewed very well but looking closely it is just the shape of it that has caused this to happen.
I improved the drawing code and also attempted to get the characters onto the same baseline:
private static Bitmap DeskewImageByIndividualChars(Bitmap bitmap)
{
IDictionary<Rectangle, Tuple<Bitmap, double>> characters = new CCL().Process(bitmap);
Bitmap deskewedBitmap = new Bitmap(bitmap.Width, bitmap.Height, bitmap.PixelFormat);
deskewedBitmap.SetResolution(bitmap.HorizontalResolution, bitmap.VerticalResolution);
using (Graphics g = Graphics.FromImage(deskewedBitmap))
{
g.FillRectangle(Brushes.White, new Rectangle(Point.Empty, deskewedBitmap.Size));
int baseLine = characters.Max(c => c.Key.Bottom);
foreach (var character in characters)
{
int y = character.Key.Y;
if (character.Key.Bottom != baseLine)
{
y += (baseLine - character.Key.Bottom - 1);
}
using (Bitmap characterBitmap = RotateImage(character.Value.Item1, character.Value.Item2, Color.White))
{
g.DrawImage(characterBitmap, new Point(character.Key.X, y));
}
}
}
return deskewedBitmap;
}
This then produces the following output. Note each character isn't on the exact same baseline due to the pre rotation bottom being taken to work it out. To improve the code using the baseline from post rotation would be needed. Also thresholding the image before doing the baseline would help.
Another improvement would be to calculate the Right of each of the rotated characters locations so when drawing the next one it doesn't overlap the previous and cut bits off. Because as you can see in the output the 2 is slightly cutting into the 5.
The output is now very similar to the manually created one in the OP.
My Application
I am writing an application that needs to convert RGB to grayscale images.
The conversion works but converting an image of 3648 * 2736 pixel takes round about 7 secs.
I know that set and getpixel take some time.
But I think that it shouldn't take so long if you are using Lockbits even though the image is not small. (please correct me if that is wrong).
Maybe I just did a fatal mistake within my code.
The code
public static long ConvertToGrayScaleV2(Bitmap imageColor, bool useHDTVConversion)
{
Stopwatch stpw = new Stopwatch();
stpw.Start();
System.Drawing.Imaging.BitmapData imageColorData = imageColor.LockBits(new Rectangle(new Point(0, 0), imageColor.Size),
System.Drawing.Imaging.ImageLockMode.ReadWrite, imageColor.PixelFormat);
IntPtr PtrColor = imageColorData.Scan0;
int strideColor = imageColorData.Stride;
byte[] byteImageColor = new byte[Math.Abs(strideColor) * imageColor.Height];
System.Runtime.InteropServices.Marshal.Copy(PtrColor, byteImageColor, 0, Math.Abs(strideColor) * imageColor.Height);
int bytesPerPixel = getBytesPerPixel(imageColor);
byte value;
if (bytesPerPixel == -1)
throw new Exception("Can't get bytes per pixel because it is not defined for this image format.");
for (int x = 0, position; x < imageColor.Width * imageColor.Height; x++)
{
position = x * bytesPerPixel;
if (useHDTVConversion)
{
value = (byte)(byteImageColor[position] * 0.0722 + byteImageColor[position + 1] * 0.7152 + byteImageColor[position + 2] * 0.2126);
}
else
{
value = (byte)(byteImageColor[position] * 0.114 + byteImageColor[position + 1] * 0.587 + byteImageColor[position + 2] * 0.299);
}
byteImageColor[position] = value;
byteImageColor[position+1] = value;
byteImageColor[position+2] = value;
}
System.Runtime.InteropServices.Marshal.Copy(byteImageColor, 0, PtrColor, Math.Abs(strideColor) * imageColor.Height);
imageColor.UnlockBits(imageColorData);
stpw.Stop();
return stpw.ElapsedMilliseconds;
}
public static int getBytesPerPixel(Image img)
{
switch (img.PixelFormat)
{
case System.Drawing.Imaging.PixelFormat.Format16bppArgb1555: return 2;
case System.Drawing.Imaging.PixelFormat.Format16bppGrayScale: return 2;
case System.Drawing.Imaging.PixelFormat.Format16bppRgb555: return 2;
case System.Drawing.Imaging.PixelFormat.Format16bppRgb565: return 2;
case System.Drawing.Imaging.PixelFormat.Format1bppIndexed: return 1;
case System.Drawing.Imaging.PixelFormat.Format24bppRgb: return 3;
case System.Drawing.Imaging.PixelFormat.Format32bppArgb: return 4;
case System.Drawing.Imaging.PixelFormat.Format32bppPArgb: return 4;
case System.Drawing.Imaging.PixelFormat.Format32bppRgb: return 4;
case System.Drawing.Imaging.PixelFormat.Format48bppRgb: return 6;
case System.Drawing.Imaging.PixelFormat.Format4bppIndexed: return 1;
case System.Drawing.Imaging.PixelFormat.Format64bppArgb: return 8;
case System.Drawing.Imaging.PixelFormat.Format64bppPArgb: return 8;
case System.Drawing.Imaging.PixelFormat.Format8bppIndexed: return 1;
default: return -1;
}
}
I know this is old, but a few possible valuable points:
The imageColor.Width * imageColor.Height is an expensive operation that you are running nearly 10 million times (3648 * 2736) more than you need to.
The for loop is recalculating that every single iteration
Not only that, but the CLR has to navigate to the Bitmap object's Width and Height properties each of those 10 million times, too. This is 30 million more operations than you need every time you try to run this on your bitmap.
Change:
for (int x = 0, position; x < imageColor.Width * imageColor.Height; x++)
{
...
}
To:
var heightWidth = imageColor.Width * imageColor.Height;
for (int x = 0, position; x < heightWidth; x++)
{
...
}
If you cache all the potential results of the three various operations (R, G, B, with 255 possible values) and use a lookup to the new values instead of calculating the new value 10 million times you'll also see a huge performance increase.
Here is the full, very fast code (much faster than a ColorMatrix). Notice I have moved all possible pre-calculated values into local variables and within the loop there absolutely minimal work involved.
var lookupR = new byte[256];
var lookupG = new byte[256];
var lookupB = new byte[256];
var rVal = hdtv ? 0.114 : 0.0722;
var gVal = hdtv ? 0.587 : 0.7152;
var bVal = hdtv ? 0.299 : 0.2126;
for (var originalValue = 0; originalValue < 256; originalValue++)
{
var r = (byte)(originalValue * rVal);
var g = (byte)(originalValue * gVal);
var b = (byte)(originalValue * bVal);
// Just in case...
if (r > 255) r = 255;
if (g > 255) g = 255;
if (b > 255) b = 255;
lookupR[originalValue] = r;
lookupG[originalValue] = g;
lookupB[originalValue] = b;
}
unsafe
{
var pointer = (byte*)(void*)bitmap.LockBits(new Rectangle(0, 0, bitmap.Width, bitmap.Height), ImageLockMode.ReadWrite, bitmap.PixelFormat);
var bytesPerPixel = getBytesPerPixel(bitmap);
var heightWidth = bitmap.Width * bitmap.Height;
for (var y = 0; y < heightWidth; ++y)
{
var value = (byte) (lookupR[pointer[0]] + lookupG[pointer[1]] + lookupB[pointer[2]]);
pointer[0] = value;
pointer[1] = value;
pointer[2] = value;
pointer += bytesPerPixel;
}
bitmap.UnlockBits();
}
break;
I ran across a similar performance issue where I need to iterate over an array of bitmap data. I found that there is a significant performance hit referencing the width or height properties of the bitmap within or as the bounds for the loop like you are doing with imagecolor.width and .height. By simply declaring an integer outside the loop and caching the bitmap height and width there in advance, I cut my loop time in half.
If you're converting to greyscale, try using a ColorMatrix transformation instead.
from: https://web.archive.org/web/20141230145627/http://bobpowell.net/grayscale.aspx
Image img = Image.FromFile(dlg.FileName);
Bitmap bm = new Bitmap(img.Width,img.Height);
Graphics g = Graphics.FromImage(bm);
ColorMatrix cm = new ColorMatrix(new float[][]{ new float[]{0.5f,0.5f,0.5f,0,0},
new float[]{0.5f,0.5f,0.5f,0,0},
new float[]{0.5f,0.5f,0.5f,0,0},
new float[]{0,0,0,1,0,0},
new float[]{0,0,0,0,1,0},
new float[]{0,0,0,0,0,1}});
/*
//Gilles Khouzams colour corrected grayscale shear
ColorMatrix cm = new ColorMatrix(new float[][]{ new float[]{0.3f,0.3f,0.3f,0,0},
new float[]{0.59f,0.59f,0.59f,0,0},
new float[]{0.11f,0.11f,0.11f,0,0},
new float[]{0,0,0,1,0,0},
new float[]{0,0,0,0,1,0},
new float[]{0,0,0,0,0,1}});
*/
ImageAttributes ia = new ImageAttributes();
ia.SetColorMatrix(cm);
g.DrawImage(img,new Rectangle(0,0,img.Width,img.Height),0,0,img.Width,img.Height,GraphicsUnit.Pixel,ia);
g.Dispose();
I would guess that the marshalling and copying is taking a large chunk of the time.
This link describes 3 methods for greyscaling an image.