From 5c6a3f2f785115adc8039e4be3242b7a9594c2d4 Mon Sep 17 00:00:00 2001 From: "Krom, Robertus" Date: Wed, 26 Feb 2020 14:59:57 +0100 Subject: [PATCH] Experimenting with some OCR and QR use cases. --- Greenshot/Forms/CaptureForm.cs | 248 ++++++++++++++---- Greenshot/Helpers/CaptureHelper.cs | 77 +++++- Greenshot/Helpers/QrExtensions.cs | 52 ++++ GreenshotPlugin/Interfaces/CaptureMode.cs | 3 +- .../Destinations/Win10OcrDestination.cs | 15 +- .../Processors/Win10OcrProcessor.cs | 8 +- GreenshotWin10Plugin/Win10OcrProvider.cs | 17 +- 7 files changed, 346 insertions(+), 74 deletions(-) create mode 100644 Greenshot/Helpers/QrExtensions.cs diff --git a/Greenshot/Forms/CaptureForm.cs b/Greenshot/Forms/CaptureForm.cs index b8e55b7be..a26d113e9 100644 --- a/Greenshot/Forms/CaptureForm.cs +++ b/Greenshot/Forms/CaptureForm.cs @@ -39,6 +39,7 @@ using System.Windows.Forms; using GreenshotPlugin.IniFile; using GreenshotPlugin.Interfaces; using GreenshotPlugin.Interfaces.Ocr; +using ZXing; namespace Greenshot.Forms { /// @@ -109,6 +110,11 @@ namespace Greenshot.Forms { private void ClosedHandler(object sender, EventArgs e) { _currentForm = null; + // Change the final mode + if (_captureMode == CaptureMode.Text) + { + _capture.CaptureDetails.CaptureMode = CaptureMode.Text; + } Log.Debug("Remove CaptureForm from currentForm"); } @@ -222,7 +228,7 @@ namespace Greenshot.Forms { Cursor.Position = new Point(Cursor.Position.X + step, Cursor.Position.Y); break; case Keys.ShiftKey: - // Fixmode + // Fix mode if (_fixMode == FixMode.None) { _fixMode = FixMode.Initiated; } @@ -278,6 +284,11 @@ namespace Greenshot.Forms { _captureRect = Rectangle.Empty; Invalidate(); break; + case CaptureMode.Text: + // Set the region capture mode + _captureMode = CaptureMode.Region; + Invalidate(); + break; case CaptureMode.Window: // Set the region capture mode _captureMode = CaptureMode.Region; @@ -306,42 +317,49 @@ namespace Greenshot.Forms { ToFront = !ToFront; TopMost = !TopMost; break; - case Keys.O: - if (_capture.CaptureDetails.OcrInformation is null) - { - var ocrProvider = SimpleServiceProvider.Current.GetInstance(); - if (ocrProvider is object) - { + case Keys.T: + _captureMode = CaptureMode.Text; + if (_capture.CaptureDetails.OcrInformation is null) + { + var ocrProvider = SimpleServiceProvider.Current.GetInstance(); + if (ocrProvider is object) + { var uiTaskScheduler = SimpleServiceProvider.Current.GetInstance(); Task.Factory.StartNew(async () => - { - _capture.CaptureDetails.OcrInformation = await ocrProvider.DoOcrAsync(_capture.Image); + { + _capture.CaptureDetails.OcrInformation = await ocrProvider.DoOcrAsync(_capture.Image); Invalidate(); - }, CancellationToken.None, TaskCreationOptions.None, uiTaskScheduler); + }, CancellationToken.None, TaskCreationOptions.None, uiTaskScheduler); + } + } + else + { + Invalidate(); + } + break; + case Keys.Q: + if (_capture.CaptureDetails.QrResult is null) + { + // create a barcode reader instance + IBarcodeReader reader = new BarcodeReader(); + // detect and decode the barcode inside the bitmap + var result = reader.Decode((Bitmap)_capture.Image); + // do something with the result + if (result != null) + { + Log.InfoFormat("Found QR of type {0} with text {1}", result.BarcodeFormat, result.Text); + _capture.CaptureDetails.QrResult = result; } } + else + { + Invalidate(); + } break; } } - /// - /// Find the list's bounding box. - /// - /// IEnumerable of Point - /// Rectangle - private Rectangle BoundingBox(IEnumerable points) - { - var x_query = from Point p in points select p.X; - int xmin = x_query.Min(); - int xmax = x_query.Max(); - - var y_query = from Point p in points select p.Y; - int ymin = y_query.Min(); - int ymax = y_query.Max(); - - return new Rectangle(xmin, ymin, xmax - xmin, ymax - ymin); - } /// /// The mousedown handler of the capture form @@ -372,18 +390,59 @@ namespace Greenshot.Forms { DialogResult = DialogResult.OK; } else if (_captureRect.Height > 0 && _captureRect.Width > 0) { // correct the GUI width to real width if Region mode - if (_captureMode == CaptureMode.Region) { + if (_captureMode == CaptureMode.Region || _captureMode == CaptureMode.Text) { _captureRect.Width += 1; _captureRect.Height += 1; } // Go and process the capture DialogResult = DialogResult.OK; + } else if (_captureMode == CaptureMode.Text && IsWordUnderCursor(_mouseMovePos)) + { + // Handle a click on a single word + _captureRect = new Rectangle(_mouseMovePos, new Size(1, 1)); + // Go and process the capture + DialogResult = DialogResult.OK; + } else if (_capture.CaptureDetails.QrResult != null && _capture.CaptureDetails.QrResult.BoundingQrBox().Contains(_mouseMovePos)) + { + // Handle a click on a QR code + _captureRect = new Rectangle(_mouseMovePos, Size.Empty); + // Go and process the capture + DialogResult = DialogResult.OK; } else { Invalidate(); } } + /// + /// + /// + /// + /// + private bool IsWordUnderCursor(Point cursorLocation) + { + if (_captureMode != CaptureMode.Text || _capture.CaptureDetails.OcrInformation == null) return false; + + var ocrInfo = _capture.CaptureDetails.OcrInformation; + + foreach (var line in ocrInfo.Lines) + { + var lineBounds = line.CalculatedBounds; + if (lineBounds.IsEmpty) continue; + // Highlight the text which is selected + if (!lineBounds.Contains(cursorLocation)) continue; + foreach (var word in line.Words) + { + if (word.Bounds.Contains(cursorLocation)) + { + return true; + } + } + } + + return false; + } + /// /// The mouse up handler of the capture form /// @@ -419,12 +478,12 @@ namespace Greenshot.Forms { /// /// The mouse move handler of the capture form /// - /// - /// + /// object + /// MouseEventArgs private void OnMouseMove(object sender, MouseEventArgs e) { // Make sure the mouse coordinates are fixed, when pressing shift - _mouseMovePos = FixMouseCoordinates(User32.GetCursorLocation()); - _mouseMovePos = WindowCapture.GetLocationRelativeToScreenBounds(_mouseMovePos); + var mouseMovePos = FixMouseCoordinates(User32.GetCursorLocation()); + _mouseMovePos = WindowCapture.GetLocationRelativeToScreenBounds(mouseMovePos); } /// @@ -461,7 +520,7 @@ namespace Greenshot.Forms { verticalMove = true; } - if (_captureMode == CaptureMode.Region && _mouseDown) { + if ((_captureMode == CaptureMode.Region || _captureMode == CaptureMode.Text) && _mouseDown) { _captureRect = GuiRectangle.GetGuiRectangle(_cursorPos.X, _cursorPos.Y, _mX - _cursorPos.X, _mY - _cursorPos.Y); } @@ -580,6 +639,62 @@ namespace Greenshot.Forms { invalidateRectangle.Offset(_cursorPos); Invalidate(invalidateRectangle); } + + // OCR + if (_captureMode == CaptureMode.Text && _capture.CaptureDetails.OcrInformation != null) + { + var ocrInfo = _capture.CaptureDetails.OcrInformation; + + invalidateRectangle = Rectangle.Empty; + foreach (var line in ocrInfo.Lines) + { + var lineBounds = line.CalculatedBounds; + if (!lineBounds.IsEmpty) + { + if (_mouseDown) + { + // Highlight the text which is selected + if (lineBounds.IntersectsWith(_captureRect)) + { + foreach (var word in line.Words) + { + if (word.Bounds.IntersectsWith(_captureRect)) + { + if (invalidateRectangle.IsEmpty) + { + invalidateRectangle = word.Bounds; + } + else + { + invalidateRectangle = Rectangle.Union(invalidateRectangle, word.Bounds); + } + } + } + } + } + else if (lineBounds.Contains(_mouseMovePos)) + { + foreach (var word in line.Words) + { + if (!word.Bounds.Contains(_mouseMovePos)) continue; + if (invalidateRectangle.IsEmpty) + { + invalidateRectangle = word.Bounds; + } + else + { + invalidateRectangle = Rectangle.Union(invalidateRectangle, word.Bounds); + } + break; + } + } + } + } + if (!invalidateRectangle.IsEmpty) + { + Invalidate(invalidateRectangle); + } + } // Force update "now" Update(); } @@ -696,7 +811,7 @@ namespace Greenshot.Forms { // Pen to draw using (Pen pen = new Pen(opacyBlack, pixelThickness)) { - // Draw the croshair-lines + // Draw the cross-hair-lines // Vertical top to middle graphics.DrawLine(pen, drawAtWidth, destinationRectangle.Y + padding, drawAtWidth, destinationRectangle.Y + halfHeightEnd - padding); // Vertical middle + 1 to bottom @@ -706,7 +821,7 @@ namespace Greenshot.Forms { // Horizontal middle + 1 to right graphics.DrawLine(pen, destinationRectangle.X + halfWidthEnd + 2 * padding, drawAtHeight, destinationRectangle.X + destinationRectangle.Width - padding, drawAtHeight); - // Fix offset for drawing the white rectangle around the crosshair-lines + // Fix offset for drawing the white rectangle around the cross-hair-lines drawAtHeight -= pixelThickness / 2; drawAtWidth -= pixelThickness / 2; // Fix off by one error with the DrawRectangle @@ -738,36 +853,65 @@ namespace Greenshot.Forms { graphics.DrawImageUnscaled(_capture.Image, Point.Empty); var ocrInfo = _capture.CaptureDetails.OcrInformation; - if (ocrInfo != null) + if (ocrInfo != null && _captureMode == CaptureMode.Text) { using var pen = new Pen(Color.Red); + var highlightColor = Color.FromArgb(128, Color.Yellow); + using var highlightTextBrush = new SolidBrush(highlightColor); foreach (var line in ocrInfo.Lines) { var lineBounds = line.CalculatedBounds; if (!lineBounds.IsEmpty) { graphics.DrawRectangle(pen, line.CalculatedBounds); - } - } - } + if (_mouseDown) + { + // Highlight the text which is selected + if (lineBounds.IntersectsWith(_captureRect)) + { + foreach (var word in line.Words) + { + if (word.Bounds.IntersectsWith(_captureRect)) + { + graphics.FillRectangle(highlightTextBrush, word.Bounds); + } + } + } + } + else if (lineBounds.Contains(_mouseMovePos)) + { + foreach (var word in line.Words) + { + if (!word.Bounds.Contains(_mouseMovePos)) continue; + graphics.FillRectangle(highlightTextBrush, word.Bounds); + break; + } + } + } + } + } + + // QR Code if (_capture.CaptureDetails.QrResult != null) { var result = _capture.CaptureDetails.QrResult; - Log.InfoFormat("Found QR of type {0} - {1}", result.BarcodeFormat, result.Text); - var boundingBox = BoundingBox(result.ResultPoints.Select(p => new Point((int)p.X, (int)p.Y))); + var boundingBox = _capture.CaptureDetails.QrResult.BoundingQrBox(); + if (!boundingBox.IsEmpty) + { + Log.InfoFormat("Found QR of type {0} - {1}", result.BarcodeFormat, result.Text); + Invalidate(boundingBox); + using var pen = new Pen(Color.BlueViolet, 10); + using var solidBrush = new SolidBrush(Color.Green); - using var pen = new Pen(Color.BlueViolet, 10); - using var solidBrush = new SolidBrush(Color.Green); - - using var solidWhiteBrush = new SolidBrush(Color.White); - using var font = new Font(FontFamily.GenericSerif, 12, FontStyle.Regular); - graphics.FillRectangle(solidWhiteBrush, boundingBox); - graphics.DrawRectangle(pen, boundingBox); - graphics.DrawString(result.Text, font, solidBrush, boundingBox); - - } + using var solidWhiteBrush = new SolidBrush(Color.White); + using var font = new Font(FontFamily.GenericSerif, 12, FontStyle.Regular); + graphics.FillRectangle(solidWhiteBrush, boundingBox); + graphics.DrawRectangle(pen, boundingBox); + graphics.DrawString(result.Text, font, solidBrush, boundingBox); + } + } // Only draw Cursor if it's (partly) visible if (_capture.Cursor != null && _capture.CursorVisible && clipRectangle.IntersectsWith(new Rectangle(_capture.CursorLocation, _capture.Cursor.Size))) { @@ -853,7 +997,7 @@ namespace Greenshot.Forms { using Font sizeFont = new Font( FontFamily.GenericSansSerif, 12 ); // When capturing a Region we need to add 1 to the height/width for correction string sizeText; - if (_captureMode == CaptureMode.Region) { + if (_captureMode == CaptureMode.Region || _captureMode == CaptureMode.Text) { // correct the GUI width to real width for the shown size sizeText = _captureRect.Width + 1 + " x " + (_captureRect.Height + 1); } else { diff --git a/Greenshot/Helpers/CaptureHelper.cs b/Greenshot/Helpers/CaptureHelper.cs index 18ddd2731..b1d7dbd3d 100644 --- a/Greenshot/Helpers/CaptureHelper.cs +++ b/Greenshot/Helpers/CaptureHelper.cs @@ -31,6 +31,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Drawing; using System.IO; +using System.Runtime.InteropServices; +using System.Text; using System.Threading; using System.Windows.Forms; using GreenshotPlugin.IniFile; @@ -201,7 +203,7 @@ namespace Greenshot.Helpers { } private void DoCaptureFeedback() { - if(CoreConfig.PlayCameraSound) { + if (CoreConfig.PlayCameraSound) { SoundHelper.Play(); } } @@ -450,7 +452,7 @@ namespace Greenshot.Helpers { Log.Warn("Unknown capture mode: " + _captureMode); break; } - // Wait for thread, otherwise we can't dipose the CaptureHelper + // Wait for thread, otherwise we can't dispose the CaptureHelper retrieveWindowDetailsThread?.Join(); if (_capture != null) { Log.Debug("Disposing capture"); @@ -546,10 +548,10 @@ namespace Greenshot.Helpers { } /// - /// This is the SufraceMessageEvent receiver + /// This is the SurfaceMessageEvent receiver /// - /// - /// + /// object + /// SurfaceMessageEventArgs private void SurfaceMessageReceived(object sender, SurfaceMessageEventArgs eventArgs) { if (string.IsNullOrEmpty(eventArgs?.Message)) { return; @@ -580,6 +582,65 @@ namespace Greenshot.Helpers { // ask to save the file as long as nothing is done. bool outputMade = false; + if (_capture.CaptureDetails.CaptureMode == CaptureMode.Text) + { + var selectionRectangle = new Rectangle(Point.Empty, _capture.Image.Size); + var ocrInfo = _capture.CaptureDetails.OcrInformation; + if (ocrInfo != null) + { + var textResult = new StringBuilder(); + foreach (var line in ocrInfo.Lines) + { + var lineBounds = line.CalculatedBounds; + if (lineBounds.IsEmpty) continue; + // Highlight the text which is selected + if (!lineBounds.IntersectsWith(selectionRectangle)) continue; + + for (var index = 0; index < line.Words.Length; index++) + { + var word = line.Words[index]; + if (!word.Bounds.IntersectsWith(selectionRectangle)) continue; + textResult.Append(word.Text); + + if (index + 1 < line.Words.Length && word.Text.Length > 1) + { + textResult.Append(' '); + } + } + + textResult.AppendLine(); + } + Clipboard.SetText(textResult.ToString()); + } + // Disable capturing + _captureMode = CaptureMode.None; + // Dispose the capture, we don't need it anymore (the surface copied all information and we got the title (if any)). + _capture.Dispose(); + _capture = null; + return; + } + + // User clicked on a QR Code + var qrResult = _capture.CaptureDetails.QrResult; + if (qrResult != null && _captureRect.Size.IsEmpty && qrResult.BoundingQrBox().Contains(_captureRect.Location)) + { + if (qrResult.Text.StartsWith("http")) + { + Process.Start(qrResult.Text); + } + else + { + Clipboard.SetText(qrResult.Text); + } + // Disable capturing + _captureMode = CaptureMode.None; + // Dispose the capture, we don't need it anymore (the surface copied all information and we got the title (if any)). + _capture.Dispose(); + _capture = null; + return; + } + + // Make sure the user sees that the capture is made if (_capture.CaptureDetails.CaptureMode == CaptureMode.File || _capture.CaptureDetails.CaptureMode == CaptureMode.Clipboard) { // Maybe not "made" but the original is still there... somehow @@ -704,7 +765,7 @@ namespace Greenshot.Helpers { Rectangle windowRectangle = windowToCapture.WindowRectangle; if (windowRectangle.Width == 0 || windowRectangle.Height == 0) { Log.WarnFormat("Window {0} has nothing to capture, using workaround to find other window of same process.", windowToCapture.Text); - // Trying workaround, the size 0 arrises with e.g. Toad.exe, has a different Window when minimized + // Trying workaround, the size 0 arises with e.g. Toad.exe, has a different Window when minimized WindowDetails linkedWindow = WindowDetails.GetLinkedWindow(windowToCapture); if (linkedWindow != null) { windowToCapture = linkedWindow; @@ -966,9 +1027,9 @@ namespace Greenshot.Helpers { Rectangle tmpRectangle = _captureRect; tmpRectangle.Offset(_capture.ScreenBounds.Location.X, _capture.ScreenBounds.Location.Y); CoreConfig.LastCapturedRegion = tmpRectangle; - HandleCapture(); } + HandleCapture(); } - } + } } } diff --git a/Greenshot/Helpers/QrExtensions.cs b/Greenshot/Helpers/QrExtensions.cs new file mode 100644 index 000000000..ac96863ad --- /dev/null +++ b/Greenshot/Helpers/QrExtensions.cs @@ -0,0 +1,52 @@ +/* + * Greenshot - a free and open source screenshot tool + * Copyright (C) 2007-2020 Thomas Braun, Jens Klingen, Robin Krom + * + * For more information see: http://getgreenshot.org/ + * The Greenshot project is hosted on GitHub https://github.com/greenshot/greenshot + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 1 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ZXing; + +namespace Greenshot.Helpers +{ + public static class QrExtensions + { + /// + /// Find the bounding box for the Qr Result. + /// + /// Result + /// Rectangle + public static Rectangle BoundingQrBox(this Result result) + { + var xValues = result.ResultPoints.Select(p => (int)p.X).ToList(); + int xMin = xValues.Min(); + int xMax = xValues.Max(); + + var yValues = result.ResultPoints.Select(p => (int)p.Y).ToList(); + int yMin = yValues.Min(); + int yMax = yValues.Max(); + + return new Rectangle(xMin, yMin, xMax - xMin, yMax - yMin); + } + } +} diff --git a/GreenshotPlugin/Interfaces/CaptureMode.cs b/GreenshotPlugin/Interfaces/CaptureMode.cs index 70dae2f32..a0aef56c4 100644 --- a/GreenshotPlugin/Interfaces/CaptureMode.cs +++ b/GreenshotPlugin/Interfaces/CaptureMode.cs @@ -35,7 +35,8 @@ namespace GreenshotPlugin.Interfaces Clipboard, File, IE, - Import + Import, + Text // Video }; } \ No newline at end of file diff --git a/GreenshotWin10Plugin/Destinations/Win10OcrDestination.cs b/GreenshotWin10Plugin/Destinations/Win10OcrDestination.cs index 53cdf22bc..617e18763 100644 --- a/GreenshotWin10Plugin/Destinations/Win10OcrDestination.cs +++ b/GreenshotWin10Plugin/Destinations/Win10OcrDestination.cs @@ -67,15 +67,20 @@ namespace GreenshotWin10Plugin.Destinations { var exportInformation = new ExportInformation(Designation, Description); try - { - var ocrProvider = SimpleServiceProvider.Current.GetInstance(); - var ocrResult = Task.Run(async () => await ocrProvider.DoOcrAsync(surface)).Result; + { + // TODO: Check if the OcrInformation is for the selected surface... otherwise discard & do it again + var ocrInformation = captureDetails.OcrInformation; + if (captureDetails.OcrInformation == null) + { + var ocrProvider = SimpleServiceProvider.Current.GetInstance(); + ocrInformation = Task.Run(async () => await ocrProvider.DoOcrAsync(surface)).Result; + } // Check if we found text - if (!string.IsNullOrWhiteSpace(ocrResult.Text)) + if (!string.IsNullOrWhiteSpace(ocrInformation.Text)) { // Place the OCR text on the - ClipboardHelper.SetClipboardData(ocrResult.Text); + ClipboardHelper.SetClipboardData(ocrInformation.Text); } exportInformation.ExportMade = true; } diff --git a/GreenshotWin10Plugin/Processors/Win10OcrProcessor.cs b/GreenshotWin10Plugin/Processors/Win10OcrProcessor.cs index 76f15429d..c5b5a8f4f 100644 --- a/GreenshotWin10Plugin/Processors/Win10OcrProcessor.cs +++ b/GreenshotWin10Plugin/Processors/Win10OcrProcessor.cs @@ -21,10 +21,8 @@ using System.Threading.Tasks; using GreenshotPlugin.Core; -using GreenshotPlugin.IniFile; using GreenshotPlugin.Interfaces; using GreenshotPlugin.Interfaces.Ocr; -using log4net; namespace GreenshotWin10Plugin.Processors { /// @@ -37,12 +35,18 @@ namespace GreenshotWin10Plugin.Processors { public override bool ProcessCapture(ISurface surface, ICaptureDetails captureDetails) { + if (captureDetails.OcrInformation != null) + { + return false; + } var ocrProvider = SimpleServiceProvider.Current.GetInstance(); + var ocrResult = Task.Run(async () => await ocrProvider.DoOcrAsync(surface)).Result; if (!ocrResult.HasContent) return false; captureDetails.OcrInformation = ocrResult; + return true; } } diff --git a/GreenshotWin10Plugin/Win10OcrProvider.cs b/GreenshotWin10Plugin/Win10OcrProvider.cs index 9dc7dc032..e8187b35e 100644 --- a/GreenshotWin10Plugin/Win10OcrProvider.cs +++ b/GreenshotWin10Plugin/Win10OcrProvider.cs @@ -57,13 +57,18 @@ namespace GreenshotWin10Plugin /// /// ISurface /// OcrResult sync - public Task DoOcrAsync(ISurface surface) + public async Task DoOcrAsync(ISurface surface) { - using var imageStream = new MemoryStream(); - ImageOutput.SaveToStream(surface, imageStream, new SurfaceOutputSettings()); - imageStream.Position = 0; - var randomAccessStream = imageStream.AsRandomAccessStream(); - return DoOcrAsync(randomAccessStream); + OcrInformation result; + using (var imageStream = new MemoryStream()) + { + ImageOutput.SaveToStream(surface, imageStream, new SurfaceOutputSettings()); + imageStream.Position = 0; + var randomAccessStream = imageStream.AsRandomAccessStream(); + + result = await DoOcrAsync(randomAccessStream); + } + return result; } ///