We Keep Coding

How do I rotate individual letters of an image into the right orientation for optimal OCR?

In my previous question, I transformed this image:
into this:
which Tesseract OCR interprets as this:
1O351
Putting a frame around the image
actually improves the OCR result.
1CB51
However, I need all 5 characters to OCR correctly, so as an experiment I used Paint.NET to rotate and align each individual letter into its proper orientation:
Resulting in the correct answer:
1CB52
How would I go about performing this correction in C#?
I've done a bit of research on various text alignment algorithms, but they all assume the existence of lines of text in the source image, lines from which you can derive a rotation angle, but which already contain the proper spacing and orientation relationships between the letters.

You can use the code in the following code project article to segment each individual character. However, when trying to deskew these characters individually any result you get is not going to be very good because there isn't very much information to go off of.
I tried using AForge.NETs HoughLineTransformation class and I got angles in the range of 80 - 90 degrees. So I tried using the following code to deskew them:
private static Bitmap DeskewImageByIndividualChars(Bitmap targetBitmap)
{
IDictionary<Rectangle, Bitmap> characters = new CCL().Process(targetBitmap);
using (Graphics g = Graphics.FromImage(targetBitmap))
{
foreach (var character in characters)
{
double angle;
BitmapData bitmapData = character.Value.LockBits(new Rectangle(Point.Empty, character.Value.Size), ImageLockMode.ReadWrite, PixelFormat.Format8bppIndexed);
try
{
HoughLineTransformation hlt = new HoughLineTransformation();
hlt.ProcessImage(bitmapData);
angle = hlt.GetLinesByRelativeIntensity(0.5).Average(l => l.Theta);
}
finally
{
character.Value.UnlockBits(bitmapData);
}
using (Bitmap bitmap = RotateImage(character.Value, 90 - angle, Color.White))
{
g.DrawImage(bitmap, character.Key.Location);
}
}
}
return targetBitmap;
}
With the RotateImage method taken from here. However, the results didn't seem to be the best. Maybe you can try and make them better.
Here is the code from the code project article for your reference. I have made a few changes to it so that it behaves a bit safer, such as adding try-finally around the LockBits and disposing of objects properly using the using statement etc.
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
namespace ConnectedComponentLabeling
{
public class CCL
{
private Bitmap _input;
private int[,] _board;
public IDictionary<Rectangle, Bitmap> Process(Bitmap input)
{
_input = input;
_board = new int[_input.Width, _input.Height];
Dictionary<int, List<Pixel>> patterns = Find();
var images = new Dictionary<Rectangle, Bitmap>();
foreach (KeyValuePair<int, List<Pixel>> pattern in patterns)
{
using (Bitmap bmp = CreateBitmap(pattern.Value))
{
images.Add(GetBounds(pattern.Value), (Bitmap)bmp.Clone());
}
}
return images;
}
protected virtual bool CheckIsBackGround(Pixel currentPixel)
{
return currentPixel.color.A == 255 && currentPixel.color.R == 255 && currentPixel.color.G == 255 && currentPixel.color.B == 255;
}
private unsafe Dictionary<int, List<Pixel>> Find()
{
int labelCount = 1;
var allLabels = new Dictionary<int, Label>();
BitmapData imageData = _input.LockBits(new Rectangle(0, 0, _input.Width, _input.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
try
{
int bytesPerPixel = 3;
byte* scan0 = (byte*)imageData.Scan0.ToPointer();
int stride = imageData.Stride;
for (int i = 0; i < _input.Height; i++)
{
byte* row = scan0 + (i * stride);
for (int j = 0; j < _input.Width; j++)
{
int bIndex = j * bytesPerPixel;
int gIndex = bIndex + 1;
int rIndex = bIndex + 2;
byte pixelR = row[rIndex];
byte pixelG = row[gIndex];
byte pixelB = row[bIndex];
Pixel currentPixel = new Pixel(new Point(j, i), Color.FromArgb(pixelR, pixelG, pixelB));
if (CheckIsBackGround(currentPixel))
{
continue;
}
IEnumerable<int> neighboringLabels = GetNeighboringLabels(currentPixel);
int currentLabel;
if (!neighboringLabels.Any())
{
currentLabel = labelCount;
allLabels.Add(currentLabel, new Label(currentLabel));
labelCount++;
}
else
{
currentLabel = neighboringLabels.Min(n => allLabels[n].GetRoot().Name);
Label root = allLabels[currentLabel].GetRoot();
foreach (var neighbor in neighboringLabels)
{
if (root.Name != allLabels[neighbor].GetRoot().Name)
{
allLabels[neighbor].Join(allLabels[currentLabel]);
}
}
}
_board[j, i] = currentLabel;
}
}
}
finally
{
_input.UnlockBits(imageData);
}
Dictionary<int, List<Pixel>> patterns = AggregatePatterns(allLabels);
patterns = RemoveIntrusions(patterns, _input.Width, _input.Height);
return patterns;
}
private Dictionary<int, List<Pixel>> RemoveIntrusions(Dictionary<int, List<Pixel>> patterns, int width, int height)
{
var patternsCleaned = new Dictionary<int, List<Pixel>>();
foreach (var pattern in patterns)
{
bool bad = false;
foreach (Pixel item in pattern.Value)
{
//Horiz
if (item.Position.X == 0)
bad = true;
else if (item.Position.Y == width - 1)
bad = true;
//Vert
else if (item.Position.Y == 0)
bad = true;
else if (item.Position.Y == height - 1)
bad = true;
}
if (!bad)
patternsCleaned.Add(pattern.Key, pattern.Value);
}
return patternsCleaned;
}
private IEnumerable<int> GetNeighboringLabels(Pixel pix)
{
var neighboringLabels = new List<int>();
for (int i = pix.Position.Y - 1; i <= pix.Position.Y + 2 && i < _input.Height - 1; i++)
{
for (int j = pix.Position.X - 1; j <= pix.Position.X + 2 && j < _input.Width - 1; j++)
{
if (i > -1 && j > -1 && _board[j, i] != 0)
{
neighboringLabels.Add(_board[j, i]);
}
}
}
return neighboringLabels;
}
private Dictionary<int, List<Pixel>> AggregatePatterns(Dictionary<int, Label> allLabels)
{
var patterns = new Dictionary<int, List<Pixel>>();
for (int i = 0; i < _input.Height; i++)
{
for (int j = 0; j < _input.Width; j++)
{
int patternNumber = _board[j, i];
if (patternNumber != 0)
{
patternNumber = allLabels[patternNumber].GetRoot().Name;
if (!patterns.ContainsKey(patternNumber))
{
patterns[patternNumber] = new List<Pixel>();
}
patterns[patternNumber].Add(new Pixel(new Point(j, i), Color.Black));
}
}
}
return patterns;
}
private unsafe Bitmap CreateBitmap(List<Pixel> pattern)
{
int minX = pattern.Min(p => p.Position.X);
int maxX = pattern.Max(p => p.Position.X);
int minY = pattern.Min(p => p.Position.Y);
int maxY = pattern.Max(p => p.Position.Y);
int width = maxX + 1 - minX;
int height = maxY + 1 - minY;
Bitmap bmp = DrawFilledRectangle(width, height);
BitmapData imageData = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height), ImageLockMode.ReadWrite, PixelFormat.Format24bppRgb);
try
{
byte* scan0 = (byte*)imageData.Scan0.ToPointer();
int stride = imageData.Stride;
foreach (Pixel pix in pattern)
{
scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride] = pix.color.B;
scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride + 1] = pix.color.G;
scan0[((pix.Position.X - minX) * 3) + (pix.Position.Y - minY) * stride + 2] = pix.color.R;
}
}
finally
{
bmp.UnlockBits(imageData);
}
return bmp;
}
private Bitmap DrawFilledRectangle(int x, int y)
{
Bitmap bmp = new Bitmap(x, y);
using (Graphics graph = Graphics.FromImage(bmp))
{
Rectangle ImageSize = new Rectangle(0, 0, x, y);
graph.FillRectangle(Brushes.White, ImageSize);
}
return bmp;
}
private Rectangle GetBounds(List<Pixel> pattern)
{
var points = pattern.Select(x => x.Position);
var x_query = points.Select(p => p.X);
int xmin = x_query.Min();
int xmax = x_query.Max();
var y_query = points.Select(p => p.Y);
int ymin = y_query.Min();
int ymax = y_query.Max();
return new Rectangle(xmin, ymin, xmax - xmin, ymax - ymin);
}
}
}
With the above code I got the following input/output:
As you can see the B has rotated quite well but the others aren't as good.
An alternative to trying to deskew the individual characters is to find there location using the segmentation routine above. Then passing each individual character through to your recognition engine separately and seeing if this improves your results.
I have used the following method to find the angle of the character using the List<Pixel> from inside the CCL class. It works by finding the angle between the "bottom left" and "bottom right" points. I haven't tested if it works if the character is rotated the other way around.
private double GetAngle(List<Pixel> pattern)
{
var pixels = pattern.Select(p => p.Position).ToArray();
Point bottomLeft = pixels.OrderByDescending(p => p.Y).ThenBy(p => p.X).First();
Point rightBottom = pixels.OrderByDescending(p => p.X).ThenByDescending(p => p.Y).First();
int xDiff = rightBottom.X - bottomLeft.X;
int yDiff = rightBottom.Y - bottomLeft.Y;
double angle = Math.Atan2(yDiff, xDiff) * 180 / Math.PI;
return -angle;
}
Note my drawing code is a bit broken so that is why the 5 is cut off on the right but this code produces the following output:
Note that the B and the 5 are rotated further than you'd expect because of their curvature.
Using the following code by getting the angle from the left and right edges and then choosing the best one, the rotations seems to be better. Note I have only tested it with letters that need rotating clockwise so if they need to go the opposite way it might not work too well.
This also "quadrants" the pixels so that each pixel is chosen from it's own quadrant as not to get two that are too nearby.
The idea in selecting the best angle is if they are similar, at the moment within 1.5 degrees of each other but can easily be updated, average them. Else we pick the one that is closest to zero.
private double GetAngle(List<Pixel> pattern, Rectangle bounds)
{
int halfWidth = bounds.X + (bounds.Width / 2);
int halfHeight = bounds.Y + (bounds.Height / 2);
double leftEdgeAngle = GetAngleLeftEdge(pattern, halfWidth, halfHeight);
double rightEdgeAngle = GetAngleRightEdge(pattern, halfWidth, halfHeight);
if (Math.Abs(leftEdgeAngle - rightEdgeAngle) <= 1.5)
{
return (leftEdgeAngle + rightEdgeAngle) / 2d;
}
if (Math.Abs(leftEdgeAngle) > Math.Abs(rightEdgeAngle))
{
return rightEdgeAngle;
}
else
{
return leftEdgeAngle;
}
}
private double GetAngleLeftEdge(List<Pixel> pattern, double halfWidth, double halfHeight)
{
var topLeftPixels = pattern.Select(p => p.Position).Where(p => p.Y < halfHeight && p.X < halfWidth).ToArray();
var bottomLeftPixels = pattern.Select(p => p.Position).Where(p => p.Y > halfHeight && p.X < halfWidth).ToArray();
Point topLeft = topLeftPixels.OrderBy(p => p.X).ThenBy(p => p.Y).First();
Point bottomLeft = bottomLeftPixels.OrderByDescending(p => p.Y).ThenBy(p => p.X).First();
int xDiff = bottomLeft.X - topLeft.X;
int yDiff = bottomLeft.Y - topLeft.Y;
double angle = Math.Atan2(yDiff, xDiff) * 180 / Math.PI;
return 90 - angle;
}
private double GetAngleRightEdge(List<Pixel> pattern, double halfWidth, double halfHeight)
{
var topRightPixels = pattern.Select(p => p.Position).Where(p => p.Y < halfHeight && p.X > halfWidth).ToArray();
var bottomRightPixels = pattern.Select(p => p.Position).Where(p => p.Y > halfHeight && p.X > halfWidth).ToArray();
Point topRight = topRightPixels.OrderBy(p => p.Y).ThenByDescending(p => p.X).First();
Point bottomRight = bottomRightPixels.OrderByDescending(p => p.X).ThenByDescending(p => p.Y).First();
int xDiff = bottomRight.X - topRight.X;
int yDiff = bottomRight.Y - topRight.Y;
double angle = Math.Atan2(xDiff, yDiff) * 180 / Math.PI;
return Math.Abs(angle);
}
This now produces the following output, again my drawing code is slightly broken. Note that the C looks to not have deskewed very well but looking closely it is just the shape of it that has caused this to happen.
I improved the drawing code and also attempted to get the characters onto the same baseline:
private static Bitmap DeskewImageByIndividualChars(Bitmap bitmap)
{
IDictionary<Rectangle, Tuple<Bitmap, double>> characters = new CCL().Process(bitmap);
Bitmap deskewedBitmap = new Bitmap(bitmap.Width, bitmap.Height, bitmap.PixelFormat);
deskewedBitmap.SetResolution(bitmap.HorizontalResolution, bitmap.VerticalResolution);
using (Graphics g = Graphics.FromImage(deskewedBitmap))
{
g.FillRectangle(Brushes.White, new Rectangle(Point.Empty, deskewedBitmap.Size));
int baseLine = characters.Max(c => c.Key.Bottom);
foreach (var character in characters)
{
int y = character.Key.Y;
if (character.Key.Bottom != baseLine)
{
y += (baseLine - character.Key.Bottom - 1);
}
using (Bitmap characterBitmap = RotateImage(character.Value.Item1, character.Value.Item2, Color.White))
{
g.DrawImage(characterBitmap, new Point(character.Key.X, y));
}
}
}
return deskewedBitmap;
}
This then produces the following output. Note each character isn't on the exact same baseline due to the pre rotation bottom being taken to work it out. To improve the code using the baseline from post rotation would be needed. Also thresholding the image before doing the baseline would help.
Another improvement would be to calculate the Right of each of the rotated characters locations so when drawing the next one it doesn't overlap the previous and cut bits off. Because as you can see in the output the 2 is slightly cutting into the 5.
The output is now very similar to the manually created one in the OP.

opencv read text in image c#(visual studio)

Hi everybody I wrote a code for text detection with opencv But I have got a mistake My code cant entire the forecah Here is my code
public int FindWrite()
{
// aimg = new IplImage(img.Size, BitDepth.U8, 1);
IplImage labelImage = new IplImage(img.Size, CvBlobLib.DepthLabel, 1);
labelImage = new IplImage(img.Size, BitDepth.U8,1);
blob = new CvBlobs();
text.Clear();
CvBlobLib.Label(labelImage,blob);
CvBlobLib.FilterByArea(blob, 600, 10000);
IplImage imgtemp = img.Clone();
// CvBlobLib.RenderBlobs(blob, img, imgtemp, RenderBlobsMode.BoundingBox | RenderBlobsMode.Angle);
// CvBlobLib.RenderBlobs(blob,img, imgtemp, RenderBlobsMode.BoundingBox | RenderBlobsMode.Angle);
CvBlobLib.RenderBlobs(blob,labelImage, imgtemp, RenderBlobsMode.BoundingBox | RenderBlobsMode.Angle);
// CvBlobLib.RenderBlobs(blob, labelImage, imgtemp, RenderBlobsMode.BoundingBox | RenderBlobsMode.Angle);
foreach (var item in blob)
{
item.Value.SetImageRoiToBlob(bimg);
// ratio values of plate between 3.5 and 5.4
double ratio = (double)item.Value.Rect.Width / item.Value.Rect.Height;
double angle = (double)item.Value.Angle();
if (ratio > 3.5 && ratio < 5.4 && angle > -15 && angle < 15)
{
IplImage texttemp = new IplImage(new CvSize(140, 27), bimg.Depth, bimg.NChannels);
Cv.Resize(bimg, texttemp);
text.Add(texttemp);
img.Rectangle(item.Value.Rect, new CvScalar(0, 0, 255), 2, LineType.Link4);
}
}
img.ResetROI();
return text.Count;
}
thanks your advance

Circular Fisheye Image dewarp to flat image

UPDATE as on 12 Nov 2015
I used PanoTools plugin with Photoshop and Hugin and played with all those parameters. End up i found the parameters for projection, HFOV and image output size that fulfill my lowest requirement.
Parameteres:
Processed Output:
My question is then how can i convert all these parameters and values into C# algorithm coding so that when I provide the original image, i will get the corrected output image?
Thanks a lot.
I have a square image captured from a circular fisheye camera. The size is 2650 * 2650 pixels.
Now, i will need to programmatically dewarp the image to a flat panorama image using C# language.
I had look around from internet with different algorithm example from Link for code below , Link1 and Link2 but just can't make it success. My maths sincerely sucks and can't help me with that. Hopefully someone able to guide me through this.
Thanks a lot.
Example of image output from the camera:
--Image grabbed from Wikipedia Fisheye Lens & size modified to fit my sample pixel.
The code i tried to dewarp it but no luck:
Bitmap sourceImage = (Bitmap)Bitmap.FromFile("circularfisheye.jpg");
double factor = 0.5;
Boolean autoCrop = false;
Color backgroundColor = Color.White;
Bitmap StartImage = null;
BitmapData srcBitmapData = null;
Byte[] srcPixels = null;
Byte[] dstPixels = null;
Bitmap NewImage = null;
BitmapData dstBitmapData = null;
try
{
// Checks whether bpp ​​( Bits Per Pixel ) is 8 , 24, or 32
int Depth = System.Drawing.Bitmap.GetPixelFormatSize(sourceImage.PixelFormat);
if (Depth != 8 && Depth != 24 && Depth != 32)
{
throw new ArgumentException("Only 8, 24 and 32 bpp images are supported.");
}
// Retrieves the count of the color components
int cCount = Depth / 8;
Size baseSize = new Size(sourceImage.Width, sourceImage.Height);
// check if a low image resize and need to improve the quality
// and not generate image aliasing
Int32 maxSize = Math.Max(sourceImage.Width, sourceImage.Height);
if (maxSize < 3000)
{
float percent = 3000F / (float)maxSize;
baseSize = new Size((Int32)((float)sourceImage.Width * percent), (Int32)((float)sourceImage.Height * percent));
}
StartImage = new Bitmap(baseSize.Width, baseSize.Height, sourceImage.PixelFormat);
StartImage.SetResolution(sourceImage.HorizontalResolution, sourceImage.VerticalResolution);
// Create the drawing object and white background
Graphics g = Graphics.FromImage(StartImage);
g.SmoothingMode = SmoothingMode.AntiAlias;
g.InterpolationMode = InterpolationMode.HighQualityBicubic;
g.PixelOffsetMode = PixelOffsetMode.HighQuality;
g.DrawImage(sourceImage, new Rectangle(-1, -1, baseSize.Width + 1, baseSize.Height + 1), 0, 0, sourceImage.Width, sourceImage.Height, GraphicsUnit.Pixel);
g.Dispose();
// Locks the source image and copies it to the byte array and releases the source image
srcBitmapData = StartImage.LockBits(new Rectangle(0, 0, StartImage.Width, StartImage.Height), ImageLockMode.ReadOnly, StartImage.PixelFormat);
srcPixels = new byte[StartImage.Width * StartImage.Height * (Depth / 8)];
Marshal.Copy(srcBitmapData.Scan0, srcPixels, 0, srcPixels.Length);
StartImage.UnlockBits(srcBitmapData);
srcBitmapData = null;
// Create the target image byte array
dstPixels = new Byte[srcPixels.Length];
// Fill the entire frame with the selected background color
Int32 index = ((1 * StartImage.Width) + 1) * cCount; //index = ((Y * Width) + X) * cCount
do
{
if (Depth == 32) //For 32 bpp defines Red , Green, Blue and Alpha
{
dstPixels[index++] = backgroundColor.B;
dstPixels[index++] = backgroundColor.G;
dstPixels[index++] = backgroundColor.R;
dstPixels[index++] = backgroundColor.A; // a
}
if (Depth == 24) //For 24 bpp defines Red , Green and Blue
{
dstPixels[index++] = backgroundColor.B;
dstPixels[index++] = backgroundColor.G;
dstPixels[index++] = backgroundColor.R;
}
if (Depth == 8)
// For 8 bpp defines the value of color ( Red , Green and Blue to be the same thing)
{
dstPixels[index++] = backgroundColor.B;
}
} while (index < srcPixels.Length);
// Calculate the maximum possible extent for the image and multiply by the desired factor
double amp = 0;
double ang = Math.PI * 0.5;
for (Int32 a = 0; a < StartImage.Height; a++)
{
int y = (int)((StartImage.Height / 2) - amp * Math.Sin(ang));
if ((y < 0) || (y > StartImage.Height))
break;
amp = a;
}
amp = (amp - 2) * (factor < -1 ? -1 : (factor > 1 ? 1 : factor));
// Define variables that calculates the cutoff points (if any)
Int32 x1, y1, x2, y2;
x1 = StartImage.Width;
y1 = StartImage.Height;
x2 = 0;
y2 = 0;
// Copy pixel by pixel for the new positions
index = ((1 * StartImage.Width) + 1) * cCount;
do
{
Int32 y = (Int32)((index / cCount) / StartImage.Width);
Int32 x = (index / cCount) - (y * StartImage.Width);
Point pt = NewPoint(new Point(x, y), StartImage.Width, StartImage.Height, amp, factor < 0);
//Values ​​for crop
if (factor >= 0)
{
if (x == StartImage.Width / 2)
{
if (pt.Y < y1)
y1 = pt.Y;
if (pt.Y > y2)
y2 = pt.Y;
}
if (y == StartImage.Height / 2)
{
if (pt.X < x1)
x1 = pt.X;
if (pt.X > x2)
x2 = pt.X;
}
}
else
{
if ((x == 1) && (y == 1))
{
y1 = pt.Y;
x1 = pt.X;
}
if ((x == StartImage.Width - 1) && (y == StartImage.Height - 1))
{
y2 = pt.Y;
x2 = pt.X;
}
}
//Bytes Index which will apply the pixel
Int32 dstIndex = ((pt.Y * StartImage.Width) + pt.X) * cCount;
if (Depth == 32)
{
dstPixels[dstIndex] = srcPixels[index++];
dstPixels[dstIndex + 1] = srcPixels[index++];
dstPixels[dstIndex + 2] = srcPixels[index++];
dstPixels[dstIndex + 3] = srcPixels[index++]; // a
}
if (Depth == 24)
{
dstPixels[dstIndex] = srcPixels[index++];
dstPixels[dstIndex + 1] = srcPixels[index++];
dstPixels[dstIndex + 2] = srcPixels[index++];
}
if (Depth == 8)
{
dstPixels[dstIndex] = srcPixels[index++];
}
} while (index < srcPixels.Length);
//Creates a new image based on the byte array previously created
NewImage = new Bitmap(StartImage.Width, StartImage.Height, StartImage.PixelFormat);
NewImage.SetResolution(StartImage.HorizontalResolution, StartImage.VerticalResolution);
dstBitmapData = NewImage.LockBits(new Rectangle(0, 0, StartImage.Width, StartImage.Height), ImageLockMode.WriteOnly, StartImage.PixelFormat);
Marshal.Copy(dstPixels, 0, dstBitmapData.Scan0, dstPixels.Length);
NewImage.UnlockBits(dstBitmapData);
//Generates the final image to crop or resize the real coo
Bitmap FinalImage = new Bitmap(sourceImage.Width + 1, sourceImage.Height, StartImage.PixelFormat);
NewImage.SetResolution(StartImage.HorizontalResolution, StartImage.VerticalResolution);
Graphics g1 = Graphics.FromImage(FinalImage);
g1.SmoothingMode = SmoothingMode.AntiAlias;
g1.InterpolationMode = InterpolationMode.HighQualityBicubic;
g1.PixelOffsetMode = PixelOffsetMode.HighQuality;
//Performs the cut if enabled automatic cutting and there is need to cut
if ((autoCrop) && ((x1 > 0) || (y1 > 0) || (x2 < NewImage.Height) || (y2 < NewImage.Height)))
{
Rectangle cropRect = new Rectangle(x1, y1, x2 - x1, y2 - y1);
g1.DrawImage(NewImage, new Rectangle(-1, -1, FinalImage.Width + 1, FinalImage.Height + 1), cropRect.X, cropRect.Y, cropRect.Width, cropRect.Height, GraphicsUnit.Pixel);
}
else
{
g1.DrawImage(NewImage, new Rectangle(-1, -1, FinalImage.Width + 1, FinalImage.Height + 1), 0, 0, NewImage.Width, NewImage.Height, GraphicsUnit.Pixel);
}
g1.Dispose();
g1 = null;
NewImage = null;
FinalImage.Save("output.jpg");
FinalImage.Dispose();
}
finally
{
srcBitmapData = null;
srcPixels = null;
dstPixels = null;
dstBitmapData = null;
}

Such a distortion as a symmetry of revolution.
In polar coordinates, with the pole at the center of the image, it is expressed as
r' = f(r)
Θ' = Θ
where the quote indicates the distorted coordinates. The function f is unknown and should be measured empirically, by calibration (looking at a regular target).
To correct the image, you need to invert the function f and apply the reverse transform to the image. In fact, it is easier to measure g directly by calibration. As a starting approximation, a simple model like
r = r' + a.r'³
can do.
Most probably you don't have a picture of a grid taken with the same lens. Your last resort is to implement the undistortion function with adjustable parameters, and optimize these by trial and error.
It should also be possible to derive the calibration curve by looking at the deformation of straight lines, but this is more "technical".
In Cartesian coordinates, you can express the correction transform as
x = g(r').x'/r'
y = g(r').y'/r'
where r' = √x'²+y'².

Use the algorithm from here:
http://www.helviojunior.com.br/fotografia/barrel-and-pincushion-distortion/
It worked for me

I've made some revamp to the HelvioJunior's library (that was linked by #Tarek.Mh), I think this may suit your need:
Below, the code:
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Runtime.InteropServices;
using static System.Math;
namespace HelvioJunior
{
//https://www.helviojunior.com.br/fotografia/barrel-and-pincushion-distortion/
public class Program
{
private static void Main(string[] args)
{
Bitmap source = (Bitmap)Image.FromFile(#"JpwX0.png");
Bitmap bmp = BarrelDistortion(source, 4/10f, true);
bmp.Save(#"test.png");
bmp.Dispose();
source.Dispose();
}
static public Bitmap BarrelDistortion(Bitmap sourceImage, double factor = 0, bool autoCrop = true, uint previewRectangleWidth = 0, Color? fillerColor = null)
{
int sourceRight = sourceImage.Width - 1, sourceBottom = sourceImage.Height - 1;
// Vertical amplitude is half the height times factor
// Horizontal amplitude is missing ; vertical amplitude's applied to both directions
double amp = sourceBottom / 2f * factor;
// Inner shrinking area points
RePoint[] lPts;
bool inverse = factor < 0;
// Shrinking area coordinates (center point is considered always available)
double x1 = sourceRight / 2f,
y1 = sourceBottom / 2f,
x2 = sourceRight / 2f,
y2 = sourceBottom / 2f;
if (inverse)
{
lPts = new RePoint[]
{
new RePoint(0, 0),
new RePoint(0, sourceBottom),
new RePoint(sourceRight, sourceBottom),
new RePoint(sourceRight, 0)
};
}
else
{
lPts = new RePoint[]
{
new RePoint(sourceRight * 1 / 2f, 0),
new RePoint(0, sourceBottom * 1 / 2f),
new RePoint(sourceRight, sourceBottom * 1 / 2f),
new RePoint(sourceRight * 1 / 2f, sourceBottom)
};
}
foreach (var pN in lPts.Select(pt => NewPoint(pt, sourceImage.Width, sourceImage.Height, amp, inverse)))
{
if (pN.Y < y1) y1 = pN.Y;
if (pN.Y > y2) y2 = pN.Y;
if (pN.X < x1) x1 = pN.X;
if (pN.X > x2) x2 = pN.X;
}
// Bytes per color from bit per pixel (bpp) format
int bpcCount = Image.GetPixelFormatSize(sourceImage.PixelFormat) / 8;
Rectangle sourceRectangle = new Rectangle(0, 0, sourceImage.Width, sourceImage.Height);
int srcLength = sourceImage.Width * sourceImage.Height * bpcCount;
// Gets sourceImage byte array as srcpixels
BitmapData srcBitmapData = sourceImage.LockBits(sourceRectangle, ImageLockMode.ReadOnly, sourceImage.PixelFormat);
byte[] srcPixels = new byte[srcLength];
Marshal.Copy(srcBitmapData.Scan0, srcPixels, 0, srcLength);
sourceImage.UnlockBits(srcBitmapData);
srcBitmapData = null;
// Destination byte array preparation as dstPixels
byte[] dstPixels = new byte[srcLength];
int dstIndex = 0;
// Filler color preparation
Color fillColor = fillerColor ?? Color.Transparent;
if (!autoCrop)
{
if (bpcCount <= 4) // Depth > 32bpp may not work as expected, filler color's not applied for bit safety reason
do
{
dstPixels[dstIndex++] = fillColor.B;
if (bpcCount > 1)
{
dstPixels[dstIndex++] = fillColor.G;
dstPixels[dstIndex++] = fillColor.R;
if (bpcCount > 3)
dstPixels[dstIndex++] = fillColor.A; // a
}
} while (dstIndex < srcLength);
}
// Byte-to-byte copy (incl. Point transformation)
int index = 0, srcBpcLength = srcLength - bpcCount;
do
{
int comp = index / bpcCount; // comp yields the current "pixel" position
int y = comp / sourceImage.Width; // Each line is sourceImage.Width bytes wide
int x = comp - (y * sourceImage.Width); // Remaining (comp - lines) bytes is target column (ranges from 0 to width - 1)
// Destination "pixel"
RePoint pt = NewPoint(new RePoint(x, y), sourceImage.Width, sourceImage.Height, amp, inverse);
dstIndex = (((int)pt.Y * sourceImage.Width) + (int)pt.X) * bpcCount; // dstIndex++ overflows when |amp| >= 2
if (dstIndex >= 0 && dstIndex <= srcBpcLength)
for (int i = 0; i++ < bpcCount;)
dstPixels[dstIndex++] = srcPixels[index++];
else
index += bpcCount;
} while (index < srcLength);
srcPixels = null;
// Destination bytes application
BitmapData dstBitmapData = sourceImage.LockBits(sourceRectangle, ImageLockMode.WriteOnly, sourceImage.PixelFormat);
Marshal.Copy(dstPixels, 0, dstBitmapData.Scan0, srcLength);
sourceImage.UnlockBits(dstBitmapData);
dstBitmapData = null;
dstPixels = null;
// Final Image area
Rectangle cropRect = new Rectangle((int)Ceiling(x1), (int)Ceiling(y1), (int)Ceiling(x2 - x1), (int)Ceiling(y2 - y1));
Rectangle destRectangle = autoCrop ? cropRect : sourceRectangle;
// Final image preparation
Bitmap FinalImage = new Bitmap(destRectangle.Width, destRectangle.Height, sourceImage.PixelFormat);
FinalImage.SetResolution(sourceImage.HorizontalResolution, sourceImage.VerticalResolution);
Graphics g1 = Graphics.FromImage(FinalImage);
g1.DrawImage(sourceImage, -destRectangle.X, -destRectangle.Y);
// Previsualization rectangle
if (previewRectangleWidth > 0)
g1.DrawRectangle(new Pen(Color.Red, previewRectangleWidth), cropRect.X - 1, cropRect.Y - 1, cropRect.Width + previewRectangleWidth, cropRect.Height + previewRectangleWidth);
g1.Dispose();
g1 = null;
return FinalImage;
}
private static RePoint NewPoint(RePoint aP, double Width, double Height, double Amplitude, bool inverse)
{
double h = aP.Y / (Height - 1);
double w = aP.X / (Width - 1);
// Works ok for [0/2] to [1/2]
// Floating point error(s) here, in the range of ]1/2] to [2/2] (No workaround found)
double sinX = Round(Sin(PI * w), 15); // Range of [0] to [1] * PI ; result ranges from 0 (far from center) to 1 (at center)
double sinY = Round(Sin(PI * h), 15);
double caX = Amplitude * (1 - 2 * w);
double caY = Amplitude * (1 - 2 * h);
double aY = 0, aX = 0;
if (inverse)
{
aX = -caX;
aY = -caY;
}
double pY = aP.Y + aY + caY * sinX;
double pX = aP.X + aX + caX * sinY;
return new RePoint(pX, pY);
}
private struct RePoint
{
public double X;
public double Y;
public RePoint(double x, double y)
{
X = x;
Y = y;
}
}
}
}

Filling the space between two rectangles

how would I go about filling the space between two rectangles when they are not vertically aligned. So if one rectangle moves up the gap between it and the next rectangle (In a chain which all follow the same path) looks smooth.
Example:
Current:
Desired:
My current drawing code:
public override void Draw(Graphics g)
{
for (int i = 0; i < this._Children.Count; i++)
{
this._Children[i].Draw(g);
if (i != 0 && i + 1 < this._Children.Count)
{
if (this._Children[i].Y != this._Children[i + 1].Y)
{
//Draw circle in gap between?
}
}
}
base.Draw(g);
}
Underlying draw:
g.FillRectangle(this.Color, this.X, this.Y, this.Size.Width, this.Size.Height);
g.DrawRectangle(this.Outline, this.X, this.Y, this.Size.Width, this.Size.Height);
Edit:
After following Jim and commenter's advice, I came up with the following. The maths seems to be wrong and it doesn't really get the effect I wanted at all though.
GraphicsPath path = new GraphicsPath();
for (int i = 0; i < this._Children.Count; i++)
{
path.AddRectangle(this._Children[i].GetBoundingBox());
if (i < this._Children.Count)
{
Block block = i == 0 ? (Block)this : this._Children[i - 1];
float x = this._Children[i].X < block.X ? this._Children[i].X : block.X;
float y = this._Children[i].Y > block.X ? this._Children[i].Y : block.Y;
float width = System.Math.Abs(this._Children[i].X - block.X);
float height = System.Math.Abs(this._Children[i].Y - block.Y);
path.AddEllipse(x, y, width, height);
}
}
g.FillPath(this._Children[0].Color, path);
g.DrawPath(this._Children[0].Outline, path);
base.Draw(g);
Edit 2: Been following everyone's advice and editing as I go along. Jim's works now but it only draws at all if you are moving up and you start moving right.
And now with TAW's advice I get ArguementInvalid, I assume this is because the rGap rectangle's height is 0.
My implementation of TAW's:
for (int i = 0; i < this._Children.Count; i++)
{
this._Children[i].Draw(g);
if (i + 1 < this._Children.Count)
{
Block block = i == 0 ? (Block)this : this._Children[i + 1];
Rectangle rec = this._Children[i].GetBoundingBox();
Rectangle rec2 = block.GetBoundingBox();
Rectangle rGap = new Rectangle(Math.Min(rec.X, rec2.X), Math.Min(rec.Y, rec2.Y), 2 * Math.Abs(rec.Left - rec2.Left), 2 * Math.Abs(rec2.Top - rec.Top));
GraphicsPath gp = new GraphicsPath();
gp.AddRectangle(rec);
gp.AddRectangle(rec2);
gp.AddArc(rGap, 0, 360);
gp.FillMode = FillMode.Winding;
g.DrawPath(this._Children[i].Outline, gp);
g.FillPath(this._Children[i].Color, gp);
}
}
base.Draw(g);
Edit 3:
I have developed my own solution after studying the problem a little more, it's not the solution I wanted but hopefully it should help someone else. Now it just needs converting to rounded corners.
Code:
for (int i = 0; i < this._Children.Count; i++)
{
this._Children[i].Draw(g);
Block block = i - 1 < 0 ? (Block)this : this._Children[i - 1];
Rectangle rec = this._Children[i].GetBoundingBox();
Rectangle rec2 = block.GetBoundingBox();
Direction dir = this._Children[i].GetDirection(true);
Direction dir2 = block.GetDirection(true);
int minX = Math.Min(rec.X, rec2.X);
int minY = Math.Min(rec.Y, rec2.Y);
int maxX = Math.Max(rec.X, rec2.X);
int maxY = Math.Max(rec.Y, rec2.Y);
int diffX = maxX - minX;
int diffY = maxY - minY;
int width = this._Children[i].Size.Width;
int height = this._Children[i].Size.Height;
Rectangle fillRec = default(Rectangle);
if ((dir == Direction.Right && dir2 == Direction.Down) || (dir == Direction.Up && dir2 == Direction.Left))
{
fillRec = new Rectangle(minX + width, minY, diffX, diffY);
}
else if ((dir == Direction.Down && dir2 == Direction.Left) || (dir == Direction.Right && dir2 == Direction.Up))
{
fillRec = new Rectangle(minX + width, (maxY + height) - diffY, diffX, diffY);
}
else if ((dir == Direction.Up && dir2 == Direction.Right) || (dir == Direction.Left && dir2 == Direction.Down))
{
fillRec = new Rectangle(minX, minY, diffX, diffY);
}
else if ((dir == Direction.Left && dir2 == Direction.Up) || (dir == Direction.Down && dir2 == Direction.Right))
{
fillRec = new Rectangle(minX, (maxY + height) - diffY, diffX, diffY);
}
if (fillRec != default(Rectangle))
{
g.FillRectangle(this._Children[i].Color, fillRec);
g.DrawRectangle(this._Children[i].Outline, fillRec);
}
}
base.Draw(g);
Produces:

Here is a solution that draws all four rounded corneres for four rectangles.
I create four rectangles overlapping with various gaps and add them and the arcs to fill the gaps to a GraphicsPath.
GraphicsPath GP = new GraphicsPath();
Rectangle fillGap(Rectangle R1, Rectangle R2, bool isTop, bool isLeft )
{
int LeftMin = Math.Min(R1.Left, R2.Left);
int RightMax = Math.Max(R1.Right, R2.Right);
int TopMin = Math.Min(R1.Top, R2.Top);
int BotMax = Math.Max(R1.Bottom, R2.Bottom);
int RightGap = 2 * Math.Abs(R1.Right - R2.Right);
int LeftGap = 2 * Math.Abs(R1.Left - R2.Left);
int TopGap = 2 * Math.Abs(R1.Top - R2.Top);
int BotGap = 2 * Math.Abs(R1.Bottom - R2.Bottom);
Rectangle R = Rectangle.Empty;
if (isTop && isLeft) R = new Rectangle(LeftMin, TopMin, LeftGap, TopGap);
if (isTop && !isLeft)
R = new Rectangle(RightMax - RightGap, TopMin, RightGap, TopGap);
if (!isTop && !isLeft)
R = new Rectangle(RightMax - RightGap, BotMax - BotGap , RightGap, BotGap );
if (!isTop && isLeft)
R = new Rectangle(LeftMin, BotMax - BotGap , LeftGap, BotGap );
return R;
}
private void button1_Click(object sender, EventArgs e)
{
Rectangle Rtop = new Rectangle(20, 10, 200, 40);
Rectangle Rbottom = new Rectangle(20, 200, 200, 40);
Rectangle Rleft = new Rectangle(10, 20, 40, 200);
Rectangle Rright = new Rectangle(210, 20, 40, 200);
GP = new GraphicsPath();
GP.FillMode = FillMode.Winding;
GP.AddRectangle(Rtop);
GP.AddRectangle(Rleft);
GP.AddRectangle(Rbottom);
GP.AddRectangle(Rright);
GP.AddArc(fillGap(Rtop, Rleft, true, true), 0, 360);
GP.AddArc(fillGap(Rtop, Rright, true, false), 0, 360);
GP.AddArc(fillGap(Rbottom, Rleft, false, true), 0, 360);
GP.AddArc(fillGap(Rbottom, Rright, false, false), 0, 360);
panel1.Invalidate();
}
private void panel1_Paint(object sender, PaintEventArgs e)
{
using (Pen pen = new Pen(Color.Black, 1.5f))
{
e.Graphics.SmoothingMode = SmoothingMode.HighQuality;
e.Graphics.DrawPath(pen, GP);
if(checkBox1.Checked) e.Graphics.FillPath(Brushes.Red, GP);
}
}
The code now assumes that your gaps are not wider than the rectanlges.
The Pen for the outline should not be smaller that 1.5f or the filling will overlap too much.
Also the Smoothing mode should be high quality, so no pixels get lost.
Here is how it looks : Drawn & filled:

You can't necessarily draw a circle in the space unless the space is square. But you can draw an ellipse.
What you need to do:
Determine extents of the rectangle in question. The center point is the point where the two rectangles meet (the bottom right part of the blank space you want to fill. The top left is the X coordinate of the left rectangle and the Y coordinate of the top rectangle. The width would be 2*(center.X - left) and the height would be 2*(center.Y - top).
Fill an ellipse in that rectangle.
Note that the above will have the effect of drawing the top-left curved portion. It won't fully fill the overlapping rectangle below.
If you want to get rid of the lines, draw a filled rectangle without a border (actually, with the border the same color as the fill color) in the overlapping space. Then draw the filled ellipse as described above, again without a border.
To draw the border of the ellipse, look into DrawArc.

How to find rectangle of difference between two images

I have two images the same size. What is the best way to find the rectangle in which they differ. Obviously I could go through the image 4 times in different directions, but i'm wondering if there's an easier way.
Example:

A naive approach would be to start at the origin, and work line by line, column by column. Compare each pixel, keeping note of the topmost, leftmost, rightmost, and bottommost, from which you can calculate your rectangle. There will be cases where this single pass approach would be faster (i.e. where there is a very small differing area)

If you want a single rectangle, use int.MaxValue for the threshold.
var diff = new ImageDiffUtil(filename1, filename2);
var diffRectangles = diff.GetDiffRectangles(int.MaxValue);
If you want multiple rectangles, use a smaller threshold.
var diff = new ImageDiffUtil(filename1, filename2);
var diffRectangles = diff.GetDiffRectangles(8);
ImageDiffUtil.cs
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
namespace diff_images
{
public class ImageDiffUtil
{
Bitmap image1;
Bitmap image2;
public ImageDiffUtil(string filename1, string filename2)
{
image1 = Image.FromFile(filename1) as Bitmap;
image2 = Image.FromFile(filename2) as Bitmap;
}
public IList<Point> GetDiffPixels()
{
var widthRange = Enumerable.Range(0, image1.Width);
var heightRange = Enumerable.Range(0, image1.Height);
var result = widthRange
.SelectMany(x => heightRange, (x, y) => new Point(x, y))
.Select(point => new
{
Point = point,
Pixel1 = image1.GetPixel(point.X, point.Y),
Pixel2 = image2.GetPixel(point.X, point.Y)
})
.Where(pair => pair.Pixel1 != pair.Pixel2)
.Select(pair => pair.Point)
.ToList();
return result;
}
public IEnumerable<Rectangle> GetDiffRectangles(double distanceThreshold)
{
var result = new List<Rectangle>();
var differentPixels = GetDiffPixels();
while (differentPixels.Count > 0)
{
var cluster = new List<Point>()
{
differentPixels[0]
};
differentPixels.RemoveAt(0);
while (true)
{
var left = cluster.Min(p => p.X);
var right = cluster.Max(p => p.X);
var top = cluster.Min(p => p.Y);
var bottom = cluster.Max(p => p.Y);
var width = Math.Max(right - left, 1);
var height = Math.Max(bottom - top, 1);
var clusterBox = new Rectangle(left, top, width, height);
var proximal = differentPixels
.Where(point => GetDistance(clusterBox, point) <= distanceThreshold)
.ToList();
proximal.ForEach(point => differentPixels.Remove(point));
if (proximal.Count == 0)
{
result.Add(clusterBox);
break;
}
else
{
cluster.AddRange(proximal);
}
};
}
return result;
}
static double GetDistance(Rectangle rect, Point p)
{
var dx = Math.Max(rect.Left - p.X, 0);
dx = Math.Max(dx, p.X - rect.Right);
var dy = Math.Max(rect.Top - p.Y, 0);
dy = Math.Max(dy, p.Y - rect.Bottom);
return Math.Sqrt(dx * dx + dy * dy);
}
}
}
Form1.cs
using System.Drawing;
using System.Linq;
using System.Windows.Forms;
namespace diff_images
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
var filename1 = #"Gelatin1.PNG";
var filename2 = #"Gelatin2.PNG";
var diff = new ImageDiffUtil(filename1, filename2);
var diffRectangles = diff.GetDiffRectangles(8);
var img3 = Image.FromFile(filename2);
Pen redPen = new Pen(Color.Red, 1);
var padding = 3;
using (var graphics = Graphics.FromImage(img3))
{
diffRectangles
.ToList()
.ForEach(rect =>
{
var largerRect = new Rectangle(rect.X - padding, rect.Y - padding, rect.Width + padding * 2, rect.Height + padding * 2);
graphics.DrawRectangle(redPen, largerRect);
});
}
var pb1 = new PictureBox()
{
Image = Image.FromFile(filename1),
Left = 8,
Top = 8,
SizeMode = PictureBoxSizeMode.AutoSize
};
var pb2 = new PictureBox()
{
Image = Image.FromFile(filename2),
Left = pb1.Left + pb1.Width + 16,
Top = 8,
SizeMode = PictureBoxSizeMode.AutoSize
};
var pb3 = new PictureBox()
{
Image = img3,
Left = pb2.Left + pb2.Width + 16,
Top = 8,
SizeMode = PictureBoxSizeMode.AutoSize
};
Controls.Add(pb1);
Controls.Add(pb2);
Controls.Add(pb3);
}
}
}

Image processing like this is expensive, there are a lot of bits to look at. In real applications, you almost always need to filter the image to get rid of artifacts induced by imperfect image captures.
A common library used for this kind of bit whacking is OpenCV, it takes advantage of dedicated CPU instructions available to make this fast. There are several .NET wrappers available for it, Emgu is one of them.

I don't think there is an easier way.
In fact doing this will just be a (very) few lines of code, so unless you find a library that does that for you directly you won't find a shorter way.

Idea:
Consider an image as a 2D Array with each Array element as a pixel of the image. Hence, I would say Image Differencing is nothing but 2D Array Differencing.
Idea is to just scan through the array elements width-wise and find the place where there is a difference in pixel values. If example [x, y] co-ordinates of both 2D Array are different then our rectangle finding logic starts. Later on the rectangles would be used to patch the last updated Frame Buffer.
We need to scan through the boundaries of the rectangles for differences and if any difference is found in the boundary of rectangle, then the boundary will be increased width-wise or height-wise depending upon the type of scan made.
Consider I scanned width-wise of 2D Array and I found a location where there exist a co-ordinate which is different in both the 2D Arrays, I will create a rectangle with the starting position as [x-1, y-1] and with the width and height as 2 and 2 respectively. Please note that width and height refers to the number of pixels.
eg: Rect Info:
X = 20
Y = 35
W = 26
H = 23
i.e width of the rectangle starts from co-ordinate [20, 35] -> [20, 35 + 26 - 1]. Maybe when you find the code you may be able to understand it better.
Also there are possibilities that there are smaller rectangles inside a bigger rectangle you have found, thus we need to remove the smaller rectangles from our reference because they mean nothing to us except that they occupu my precious space !!
The above logic would be helpful in the case of VNC Server Implementation where there would be a need of rectangles that denotes differences in the image that is currently taken. Those rectangles could be sent in the network to the VNC Client which can patch the rectangles in the local copy of Frame Buffer it possesses thereby displaying it on the VNC Client Display Board.
P.S.:
I will be attaching the code in which I implemented my own algorithm. I would request viewers to comment for any mistakes or performance tuning. I would also request viewers to comment about any better algorithm that would make life simpler.
Code:
Class Rect:
public class Rect {
public int x; // Array Index
public int y; // Array Index
public int w; // Number of hops along the Horizontal
public int h; // Number of hops along the Vertical
#Override
public boolean equals(Object obj) {
Rect rect = (Rect) obj;
if(rect.x == this.x && rect.y == this.y && rect.w == this.w && rect.h == this.h) {
return true;
}
return false;
}
}
Class Image Difference:
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import javax.imageio.ImageIO;
public class ImageDifference {
long start = 0, end = 0;
public LinkedList<Rect> differenceImage(int[][] baseFrame, int[][] screenShot, int xOffset, int yOffset, int width, int height) {
// Code starts here
int xRover = 0;
int yRover = 0;
int index = 0;
int limit = 0;
int rover = 0;
boolean isRectChanged = false;
boolean shouldSkip = false;
LinkedList<Rect> rectangles = new LinkedList<Rect>();
Rect rect = null;
start = System.nanoTime();
// xRover - Rovers over the height of 2D Array
// yRover - Rovers over the width of 2D Array
int verticalLimit = xOffset + height;
int horizontalLimit = yOffset + width;
for(xRover = xOffset; xRover < verticalLimit; xRover += 1) {
for(yRover = yOffset; yRover < horizontalLimit; yRover += 1) {
if(baseFrame[xRover][yRover] != screenShot[xRover][yRover]) {
// Skip over the already processed Rectangles
for(Rect itrRect : rectangles) {
if(( (xRover < itrRect.x + itrRect.h) && (xRover >= itrRect.x) ) && ( (yRover < itrRect.y + itrRect.w) && (yRover >= itrRect.y) )) {
shouldSkip = true;
yRover = itrRect.y + itrRect.w - 1;
break;
} // End if(( (xRover < itrRect.x + itrRect.h) && (xRover >= itrRect.x) ) && ( (yRover < itrRect.y + itrRect.w) && (yRover >= itrRect.y) ))
} // End for(Rect itrRect : rectangles)
if(shouldSkip) {
shouldSkip = false;
// Need to come out of the if condition as below that is why "continue" has been provided
// if(( (xRover <= (itrRect.x + itrRect.h)) && (xRover >= itrRect.x) ) && ( (yRover <= (itrRect.y + itrRect.w)) && (yRover >= itrRect.y) ))
continue;
} // End if(shouldSkip)
rect = new Rect();
rect.x = ((xRover - 1) < xOffset) ? xOffset : (xRover - 1);
rect.y = ((yRover - 1) < yOffset) ? yOffset : (yRover - 1);
rect.w = 2;
rect.h = 2;
/* Boolean variable used to re-scan the currently found rectangle
for any change due to previous scanning of boundaries */
isRectChanged = true;
while(isRectChanged) {
isRectChanged = false;
index = 0;
/* I */
/* Scanning of left-side boundary of rectangle */
index = rect.x;
limit = rect.x + rect.h;
while(index < limit && rect.y != yOffset) {
if(baseFrame[index][rect.y] != screenShot[index][rect.y]) {
isRectChanged = true;
rect.y = rect.y - 1;
rect.w = rect.w + 1;
index = rect.x;
continue;
} // End if(baseFrame[index][rect.y] != screenShot[index][rect.y])
index = index + 1;;
} // End while(index < limit && rect.y != yOffset)
/* II */
/* Scanning of bottom boundary of rectangle */
index = rect.y;
limit = rect.y + rect.w;
while( (index < limit) && (rect.x + rect.h != verticalLimit) ) {
rover = rect.x + rect.h - 1;
if(baseFrame[rover][index] != screenShot[rover][index]) {
isRectChanged = true;
rect.h = rect.h + 1;
index = rect.y;
continue;
} // End if(baseFrame[rover][index] != screenShot[rover][index])
index = index + 1;
} // End while( (index < limit) && (rect.x + rect.h != verticalLimit) )
/* III */
/* Scanning of right-side boundary of rectangle */
index = rect.x;
limit = rect.x + rect.h;
while( (index < limit) && (rect.y + rect.w != horizontalLimit) ) {
rover = rect.y + rect.w - 1;
if(baseFrame[index][rover] != screenShot[index][rover]) {
isRectChanged = true;
rect.w = rect.w + 1;
index = rect.x;
continue;
} // End if(baseFrame[index][rover] != screenShot[index][rover])
index = index + 1;
} // End while( (index < limit) && (rect.y + rect.w != horizontalLimit) )
} // while(isRectChanged)
// Remove those rectangles that come inside "rect" rectangle.
int idx = 0;
while(idx < rectangles.size()) {
Rect r = rectangles.get(idx);
if( ( (rect.x <= r.x) && (rect.x + rect.h >= r.x + r.h) ) && ( (rect.y <= r.y) && (rect.y + rect.w >= r.y + r.w) ) ) {
rectangles.remove(r);
} else {
idx += 1;
} // End if( ( (rect.x <= r.x) && (rect.x + rect.h >= r.x + r.h) ) && ( (rect.y <= r.y) && (rect.y + rect.w >= r.y + r.w) ) )
} // End while(idx < rectangles.size())
// Giving a head start to the yRover when a rectangle is found
rectangles.addFirst(rect);
yRover = rect.y + rect.w - 1;
rect = null;
} // End if(baseFrame[xRover][yRover] != screenShot[xRover][yRover])
} // End for(yRover = yOffset; yRover < horizontalLimit; yRover += 1)
} // End for(xRover = xOffset; xRover < verticalLimit; xRover += 1)
end = System.nanoTime();
return rectangles;
}
public static void main(String[] args) throws IOException {
LinkedList<Rect> rectangles = null;
// Buffering the Base image and Screen Shot Image
BufferedImage screenShotImg = ImageIO.read(new File("screenShotImg.png"));
BufferedImage baseImg = ImageIO.read(new File("baseImg.png"));
int width = baseImg.getWidth();
int height = baseImg.getHeight();
int xOffset = 0;
int yOffset = 0;
int length = baseImg.getWidth() * baseImg.getHeight();
// Creating 2 Two Dimensional Arrays for Image Processing
int[][] baseFrame = new int[height][width];
int[][] screenShot = new int[height][width];
// Creating 2 Single Dimensional Arrays to retrieve the Pixel Values
int[] baseImgPix = new int[length];
int[] screenShotImgPix = new int[length];
// Reading the Pixels from the Buffered Image
baseImg.getRGB(0, 0, baseImg.getWidth(), baseImg.getHeight(), baseImgPix, 0, baseImg.getWidth());
screenShotImg.getRGB(0, 0, screenShotImg.getWidth(), screenShotImg.getHeight(), screenShotImgPix, 0, screenShotImg.getWidth());
// Transporting the Single Dimensional Arrays to Two Dimensional Array
long start = System.nanoTime();
for(int row = 0; row < height; row++) {
System.arraycopy(baseImgPix, (row * width), baseFrame[row], 0, width);
System.arraycopy(screenShotImgPix, (row * width), screenShot[row], 0, width);
}
long end = System.nanoTime();
System.out.println("Array Copy : " + ((double)(end - start) / 1000000));
// Finding Differences between the Base Image and ScreenShot Image
ImageDifference imDiff = new ImageDifference();
rectangles = imDiff.differenceImage(baseFrame, screenShot, xOffset, yOffset, width, height);
// Displaying the rectangles found
int index = 0;
for(Rect rect : rectangles) {
System.out.println("\nRect info : " + (++index));
System.out.println("X : " + rect.x);
System.out.println("Y : " + rect.y);
System.out.println("W : " + rect.w);
System.out.println("H : " + rect.h);
// Creating Bounding Box
for(int i = rect.y; i < rect.y + rect.w; i++) {
screenShotImgPix[ ( rect.x * width) + i ] = 0xFFFF0000;
screenShotImgPix[ ((rect.x + rect.h - 1) * width) + i ] = 0xFFFF0000;
}
for(int j = rect.x; j < rect.x + rect.h; j++) {
screenShotImgPix[ (j * width) + rect.y ] = 0xFFFF0000;
screenShotImgPix[ (j * width) + (rect.y + rect.w - 1) ] = 0xFFFF0000;
}
}
// Creating the Resultant Image
screenShotImg.setRGB(0, 0, width, height, screenShotImgPix, 0, width);
ImageIO.write(screenShotImg, "PNG", new File("result.png"));
double d = ((double)(imDiff.end - imDiff.start) / 1000000);
System.out.println("\nTotal Time : " + d + " ms" + " Array Copy : " + ((double)(end - start) / 1000000) + " ms");
}
}
Description:
There would be a function named
public LinkedList<Rect> differenceImage(int[][] baseFrame, int[][] screenShot, int width, int height)
which does the job of finding differences in the images and return a linkedlist of objects. The objects are nothing but the rectangles.
There is main function which does the job of testing the algorithm.
There are 2 sample images passed into the code in main function, they are nothing but the "baseFrame" and "screenShot" thereby creating the resultant image named "result".
I don't possess the desired reputation to post the resultant image which would be very interesting.
There is a blog which would provide the output
Image Difference

I don't think there can be anything better than exhaustively searching from each side in turn for the first point of difference in that direction. Unless, that is, you know a fact that in some way constrains the set of points of difference.

So here comes the easy way if you know how to use Lockbit :)
Bitmap originalBMP = new Bitmap(pictureBox1.ImageLocation);
Bitmap changedBMP = new Bitmap(pictureBox2.ImageLocation);
int width = Math.Min(originalBMP.Width, changedBMP.Width),
height = Math.Min(originalBMP.Height, changedBMP.Height),
xMin = int.MaxValue,
xMax = int.MinValue,
yMin = int.MaxValue,
yMax = int.MinValue;
var originalLock = originalBMP.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadWrite, originalBMP.PixelFormat);
var changedLock = changedBMP.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadWrite, changedBMP.PixelFormat);
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
//generate the address of the colour pixel
int pixelIdxOrg = y * originalLock.Stride + (x * 4);
int pixelIdxCh = y * changedLock.Stride + (x * 4);
if (( Marshal.ReadByte(originalLock.Scan0, pixelIdxOrg + 2)!= Marshal.ReadByte(changedLock.Scan0, pixelIdxCh + 2))
|| (Marshal.ReadByte(originalLock.Scan0, pixelIdxOrg + 1) != Marshal.ReadByte(changedLock.Scan0, pixelIdxCh + 1))
|| (Marshal.ReadByte(originalLock.Scan0, pixelIdxOrg) != Marshal.ReadByte(changedLock.Scan0, pixelIdxCh))
)
{
xMin = Math.Min(xMin, x);
xMax = Math.Max(xMax, x);
yMin = Math.Min(yMin, y);
yMax = Math.Max(yMax, y);
}
}
}
originalBMP.UnlockBits(originalLock);
changedBMP.UnlockBits(changedLock);
var result = changedBMP.Clone(new Rectangle(xMin, yMin, xMax - xMin, yMax - yMin), changedBMP.PixelFormat);
pictureBox3.Image = result;
disclaim it looks like your 2 pictures contains more differences than we can see with the naked eye so the result will be wider than you expect but you can add a tolerance so it wil fit even if the rest isn't 100% identical
to speed things up you will maybe able to us Parallel.For but do it only for the outer loop