github/greenshot
1
0
Fork 0
mirror of https://github.com/greenshot/greenshot synced 2025-08-19 21:13:23 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Experimenting with some OCR and QR use cases.

Browse source
This commit is contained in:
Krom, Robertus 2020-02-26 14:59:57 +01:00
commit 5c6a3f2f78
7 changed files with 346 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

208
Greenshot/Forms/CaptureForm.cs
View file

@ -39,6 +39,7 @@ using System.Windows.Forms;
using GreenshotPlugin.IniFile;
using GreenshotPlugin.Interfaces;
using GreenshotPlugin.Interfaces.Ocr;
using ZXing;
namespace Greenshot.Forms {
/// <summary>
@ -109,6 +110,11 @@ namespace Greenshot.Forms {
private void ClosedHandler(object sender, EventArgs e) {
_currentForm = null;
// Change the final mode
if (_captureMode == CaptureMode.Text)
{
_capture.CaptureDetails.CaptureMode = CaptureMode.Text;
}
Log.Debug("Remove CaptureForm from currentForm");
}
@ -222,7 +228,7 @@ namespace Greenshot.Forms {
Cursor.Position = new Point(Cursor.Position.X + step, Cursor.Position.Y);
break;
case Keys.ShiftKey:
// Fixmode
// Fix mode
if (_fixMode == FixMode.None) {
_fixMode = FixMode.Initiated;
}
@ -278,6 +284,11 @@ namespace Greenshot.Forms {
_captureRect = Rectangle.Empty;
Invalidate();
break;
case CaptureMode.Text:
// Set the region capture mode
_captureMode = CaptureMode.Region;
Invalidate();
break;
case CaptureMode.Window:
// Set the region capture mode
_captureMode = CaptureMode.Region;
@ -306,7 +317,8 @@ namespace Greenshot.Forms {
ToFront = !ToFront;
TopMost = !TopMost;
break;
case Keys.O:
case Keys.T:
_captureMode = CaptureMode.Text;
if (_capture.CaptureDetails.OcrInformation is null)
{
var ocrProvider = SimpleServiceProvider.Current.GetInstance<IOcrProvider>();
@ -321,27 +333,33 @@ namespace Greenshot.Forms {
}, CancellationToken.None, TaskCreationOptions.None, uiTaskScheduler);
}
}
else
{
Invalidate();
}
break;
case Keys.Q:
if (_capture.CaptureDetails.QrResult is null)
{
// create a barcode reader instance
IBarcodeReader reader = new BarcodeReader();
// detect and decode the barcode inside the bitmap
var result = reader.Decode((Bitmap)_capture.Image);
// do something with the result
if (result != null)
{
Log.InfoFormat("Found QR of type {0} with text {1}", result.BarcodeFormat, result.Text);
_capture.CaptureDetails.QrResult = result;
}
}
else
{
Invalidate();
}
break;
}
}
/// <summary>
/// Find the list's bounding box.
/// </summary>
/// <param name="points">IEnumerable of Point</param>
/// <returns>Rectangle</returns>
private Rectangle BoundingBox(IEnumerable<Point> points)
{
var x_query = from Point p in points select p.X;
int xmin = x_query.Min();
int xmax = x_query.Max();
var y_query = from Point p in points select p.Y;
int ymin = y_query.Min();
int ymax = y_query.Max();
return new Rectangle(xmin, ymin, xmax - xmin, ymax - ymin);
}
/// <summary>
/// The mousedown handler of the capture form
@ -372,18 +390,59 @@ namespace Greenshot.Forms {
DialogResult = DialogResult.OK;
} else if (_captureRect.Height > 0 && _captureRect.Width > 0) {
// correct the GUI width to real width if Region mode
if (_captureMode == CaptureMode.Region) {
if (_captureMode == CaptureMode.Region || _captureMode == CaptureMode.Text) {
_captureRect.Width += 1;
_captureRect.Height += 1;
}
// Go and process the capture
DialogResult = DialogResult.OK;
} else if (_captureMode == CaptureMode.Text && IsWordUnderCursor(_mouseMovePos))
{
// Handle a click on a single word
_captureRect = new Rectangle(_mouseMovePos, new Size(1, 1));
// Go and process the capture
DialogResult = DialogResult.OK;
} else if (_capture.CaptureDetails.QrResult != null && _capture.CaptureDetails.QrResult.BoundingQrBox().Contains(_mouseMovePos))
{
// Handle a click on a QR code
_captureRect = new Rectangle(_mouseMovePos, Size.Empty);
// Go and process the capture
DialogResult = DialogResult.OK;
} else {
Invalidate();
}
}
/// <summary>
///
/// </summary>
/// <param name="cursorLocation"></param>
/// <returns></returns>
private bool IsWordUnderCursor(Point cursorLocation)
{
if (_captureMode != CaptureMode.Text || _capture.CaptureDetails.OcrInformation == null) return false;
var ocrInfo = _capture.CaptureDetails.OcrInformation;
foreach (var line in ocrInfo.Lines)
{
var lineBounds = line.CalculatedBounds;
if (lineBounds.IsEmpty) continue;
// Highlight the text which is selected
if (!lineBounds.Contains(cursorLocation)) continue;
foreach (var word in line.Words)
{
if (word.Bounds.Contains(cursorLocation))
{
return true;
}
}
}
return false;
}
/// <summary>
/// The mouse up handler of the capture form
/// </summary>
@ -419,12 +478,12 @@ namespace Greenshot.Forms {
/// <summary>
/// The mouse move handler of the capture form
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
/// <param name="sender">object</param>
/// <param name="e">MouseEventArgs</param>
private void OnMouseMove(object sender, MouseEventArgs e) {
// Make sure the mouse coordinates are fixed, when pressing shift
_mouseMovePos = FixMouseCoordinates(User32.GetCursorLocation());
_mouseMovePos = WindowCapture.GetLocationRelativeToScreenBounds(_mouseMovePos);
var mouseMovePos = FixMouseCoordinates(User32.GetCursorLocation());
_mouseMovePos = WindowCapture.GetLocationRelativeToScreenBounds(mouseMovePos);
}
/// <summary>
@ -461,7 +520,7 @@ namespace Greenshot.Forms {
verticalMove = true;
}
if (_captureMode == CaptureMode.Region && _mouseDown) {
if ((_captureMode == CaptureMode.Region || _captureMode == CaptureMode.Text) && _mouseDown) {
_captureRect = GuiRectangle.GetGuiRectangle(_cursorPos.X, _cursorPos.Y, _mX - _cursorPos.X, _mY - _cursorPos.Y);
}
@ -580,6 +639,62 @@ namespace Greenshot.Forms {
invalidateRectangle.Offset(_cursorPos);
Invalidate(invalidateRectangle);
}
// OCR
if (_captureMode == CaptureMode.Text && _capture.CaptureDetails.OcrInformation != null)
{
var ocrInfo = _capture.CaptureDetails.OcrInformation;
invalidateRectangle = Rectangle.Empty;
foreach (var line in ocrInfo.Lines)
{
var lineBounds = line.CalculatedBounds;
if (!lineBounds.IsEmpty)
{
if (_mouseDown)
{
// Highlight the text which is selected
if (lineBounds.IntersectsWith(_captureRect))
{
foreach (var word in line.Words)
{
if (word.Bounds.IntersectsWith(_captureRect))
{
if (invalidateRectangle.IsEmpty)
{
invalidateRectangle = word.Bounds;
}
else
{
invalidateRectangle = Rectangle.Union(invalidateRectangle, word.Bounds);
}
}
}
}
}
else if (lineBounds.Contains(_mouseMovePos))
{
foreach (var word in line.Words)
{
if (!word.Bounds.Contains(_mouseMovePos)) continue;
if (invalidateRectangle.IsEmpty)
{
invalidateRectangle = word.Bounds;
}
else
{
invalidateRectangle = Rectangle.Union(invalidateRectangle, word.Bounds);
}
break;
}
}
}
}
if (!invalidateRectangle.IsEmpty)
{
Invalidate(invalidateRectangle);
}
}
// Force update "now"
Update();
}
@ -696,7 +811,7 @@ namespace Greenshot.Forms {
// Pen to draw
using (Pen pen = new Pen(opacyBlack, pixelThickness)) {
// Draw the croshair-lines
// Draw the cross-hair-lines
// Vertical top to middle
graphics.DrawLine(pen, drawAtWidth, destinationRectangle.Y + padding, drawAtWidth, destinationRectangle.Y + halfHeightEnd - padding);
// Vertical middle + 1 to bottom
@ -706,7 +821,7 @@ namespace Greenshot.Forms {
// Horizontal middle + 1 to right
graphics.DrawLine(pen, destinationRectangle.X + halfWidthEnd + 2 * padding, drawAtHeight, destinationRectangle.X + destinationRectangle.Width - padding, drawAtHeight);
// Fix offset for drawing the white rectangle around the crosshair-lines
// Fix offset for drawing the white rectangle around the cross-hair-lines
drawAtHeight -= pixelThickness / 2;
drawAtWidth -= pixelThickness / 2;
// Fix off by one error with the DrawRectangle
@ -738,26 +853,55 @@ namespace Greenshot.Forms {
graphics.DrawImageUnscaled(_capture.Image, Point.Empty);
var ocrInfo = _capture.CaptureDetails.OcrInformation;
if (ocrInfo != null)
if (ocrInfo != null && _captureMode == CaptureMode.Text)
{
using var pen = new Pen(Color.Red);
var highlightColor = Color.FromArgb(128, Color.Yellow);
using var highlightTextBrush = new SolidBrush(highlightColor);
foreach (var line in ocrInfo.Lines)
{
var lineBounds = line.CalculatedBounds;
if (!lineBounds.IsEmpty)
{
graphics.DrawRectangle(pen, line.CalculatedBounds);
if (_mouseDown)
{
// Highlight the text which is selected
if (lineBounds.IntersectsWith(_captureRect))
{
foreach (var word in line.Words)
{
if (word.Bounds.IntersectsWith(_captureRect))
{
graphics.FillRectangle(highlightTextBrush, word.Bounds);
}
}
}
}
else if (lineBounds.Contains(_mouseMovePos))
{
foreach (var word in line.Words)
{
if (!word.Bounds.Contains(_mouseMovePos)) continue;
graphics.FillRectangle(highlightTextBrush, word.Bounds);
break;
}
}
}
}
}
// QR Code
if (_capture.CaptureDetails.QrResult != null)
{
var result = _capture.CaptureDetails.QrResult;
var boundingBox = _capture.CaptureDetails.QrResult.BoundingQrBox();
if (!boundingBox.IsEmpty)
{
Log.InfoFormat("Found QR of type {0} - {1}", result.BarcodeFormat, result.Text);
var boundingBox = BoundingBox(result.ResultPoints.Select(p => new Point((int)p.X, (int)p.Y)));
Invalidate(boundingBox);
using var pen = new Pen(Color.BlueViolet, 10);
using var solidBrush = new SolidBrush(Color.Green);
@ -766,7 +910,7 @@ namespace Greenshot.Forms {
graphics.FillRectangle(solidWhiteBrush, boundingBox);
graphics.DrawRectangle(pen, boundingBox);
graphics.DrawString(result.Text, font, solidBrush, boundingBox);
}
}
// Only draw Cursor if it's (partly) visible
@ -853,7 +997,7 @@ namespace Greenshot.Forms {
using Font sizeFont = new Font( FontFamily.GenericSansSerif, 12 );
// When capturing a Region we need to add 1 to the height/width for correction
string sizeText;
if (_captureMode == CaptureMode.Region) {
if (_captureMode == CaptureMode.Region || _captureMode == CaptureMode.Text) {
// correct the GUI width to real width for the shown size
sizeText = _captureRect.Width + 1 + " x " + (_captureRect.Height + 1);
} else {

75
Greenshot/Helpers/CaptureHelper.cs
View file

@ -31,6 +31,8 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Windows.Forms;
using GreenshotPlugin.IniFile;
@ -201,7 +203,7 @@ namespace Greenshot.Helpers {
}
private void DoCaptureFeedback() {
if(CoreConfig.PlayCameraSound) {
if (CoreConfig.PlayCameraSound) {
SoundHelper.Play();
}
}
@ -450,7 +452,7 @@ namespace Greenshot.Helpers {
Log.Warn("Unknown capture mode: " + _captureMode);
break;
}
// Wait for thread, otherwise we can't dipose the CaptureHelper
// Wait for thread, otherwise we can't dispose the CaptureHelper
retrieveWindowDetailsThread?.Join();
if (_capture != null) {
Log.Debug("Disposing capture");
@ -546,10 +548,10 @@ namespace Greenshot.Helpers {
}
/// <summary>
/// This is the SufraceMessageEvent receiver
/// This is the SurfaceMessageEvent receiver
/// </summary>
/// <param name="sender"></param>
/// <param name="eventArgs"></param>
/// <param name="sender">object</param>
/// <param name="eventArgs">SurfaceMessageEventArgs</param>
private void SurfaceMessageReceived(object sender, SurfaceMessageEventArgs eventArgs) {
if (string.IsNullOrEmpty(eventArgs?.Message)) {
return;
@ -580,6 +582,65 @@ namespace Greenshot.Helpers {
// ask to save the file as long as nothing is done.
bool outputMade = false;
if (_capture.CaptureDetails.CaptureMode == CaptureMode.Text)
{
var selectionRectangle = new Rectangle(Point.Empty, _capture.Image.Size);
var ocrInfo = _capture.CaptureDetails.OcrInformation;
if (ocrInfo != null)
{
var textResult = new StringBuilder();
foreach (var line in ocrInfo.Lines)
{
var lineBounds = line.CalculatedBounds;
if (lineBounds.IsEmpty) continue;
// Highlight the text which is selected
if (!lineBounds.IntersectsWith(selectionRectangle)) continue;
for (var index = 0; index < line.Words.Length; index++)
{
var word = line.Words[index];
if (!word.Bounds.IntersectsWith(selectionRectangle)) continue;
textResult.Append(word.Text);
if (index + 1 < line.Words.Length && word.Text.Length > 1)
{
textResult.Append(' ');
}
}
textResult.AppendLine();
}
Clipboard.SetText(textResult.ToString());
}
// Disable capturing
_captureMode = CaptureMode.None;
// Dispose the capture, we don't need it anymore (the surface copied all information and we got the title (if any)).
_capture.Dispose();
_capture = null;
return;
}
// User clicked on a QR Code
var qrResult = _capture.CaptureDetails.QrResult;
if (qrResult != null && _captureRect.Size.IsEmpty && qrResult.BoundingQrBox().Contains(_captureRect.Location))
{
if (qrResult.Text.StartsWith("http"))
{
Process.Start(qrResult.Text);
}
else
{
Clipboard.SetText(qrResult.Text);
}
// Disable capturing
_captureMode = CaptureMode.None;
// Dispose the capture, we don't need it anymore (the surface copied all information and we got the title (if any)).
_capture.Dispose();
_capture = null;
return;
}
// Make sure the user sees that the capture is made
if (_capture.CaptureDetails.CaptureMode == CaptureMode.File || _capture.CaptureDetails.CaptureMode == CaptureMode.Clipboard) {
// Maybe not "made" but the original is still there... somehow
@ -704,7 +765,7 @@ namespace Greenshot.Helpers {
Rectangle windowRectangle = windowToCapture.WindowRectangle;
if (windowRectangle.Width == 0 || windowRectangle.Height == 0) {
Log.WarnFormat("Window {0} has nothing to capture, using workaround to find other window of same process.", windowToCapture.Text);
// Trying workaround, the size 0 arrises with e.g. Toad.exe, has a different Window when minimized
// Trying workaround, the size 0 arises with e.g. Toad.exe, has a different Window when minimized
WindowDetails linkedWindow = WindowDetails.GetLinkedWindow(windowToCapture);
if (linkedWindow != null) {
windowToCapture = linkedWindow;
@ -966,9 +1027,9 @@ namespace Greenshot.Helpers {
Rectangle tmpRectangle = _captureRect;
tmpRectangle.Offset(_capture.ScreenBounds.Location.X, _capture.ScreenBounds.Location.Y);
CoreConfig.LastCapturedRegion = tmpRectangle;
}
HandleCapture();
}
}
}
}
}

52
Greenshot/Helpers/QrExtensions.cs Normal file
View file

@ -0,0 +1,52 @@
/*
* Greenshot - a free and open source screenshot tool
* Copyright (C) 2007-2020 Thomas Braun, Jens Klingen, Robin Krom
*
* For more information see: http://getgreenshot.org/
* The Greenshot project is hosted on GitHub https://github.com/greenshot/greenshot
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ZXing;
namespace Greenshot.Helpers
{
public static class QrExtensions
{
/// <summary>
/// Find the bounding box for the Qr Result.
/// </summary>
/// <param name="result">Result</param>
/// <returns>Rectangle</returns>
public static Rectangle BoundingQrBox(this Result result)
{
var xValues = result.ResultPoints.Select(p => (int)p.X).ToList();
int xMin = xValues.Min();
int xMax = xValues.Max();
var yValues = result.ResultPoints.Select(p => (int)p.Y).ToList();
int yMin = yValues.Min();
int yMax = yValues.Max();
return new Rectangle(xMin, yMin, xMax - xMin, yMax - yMin);
}
}
}

3
GreenshotPlugin/Interfaces/CaptureMode.cs
View file

@ -35,7 +35,8 @@ namespace GreenshotPlugin.Interfaces
Clipboard,
File,
IE,
Import
Import,
Text
// Video
};
}

11
GreenshotWin10Plugin/Destinations/Win10OcrDestination.cs
View file

@ -67,15 +67,20 @@ namespace GreenshotWin10Plugin.Destinations
{
var exportInformation = new ExportInformation(Designation, Description);
try
{
// TODO: Check if the OcrInformation is for the selected surface... otherwise discard & do it again
var ocrInformation = captureDetails.OcrInformation;
if (captureDetails.OcrInformation == null)
{
var ocrProvider = SimpleServiceProvider.Current.GetInstance<IOcrProvider>();
var ocrResult = Task.Run(async () => await ocrProvider.DoOcrAsync(surface)).Result;
ocrInformation = Task.Run(async () => await ocrProvider.DoOcrAsync(surface)).Result;
}
// Check if we found text
if (!string.IsNullOrWhiteSpace(ocrResult.Text))
if (!string.IsNullOrWhiteSpace(ocrInformation.Text))
{
// Place the OCR text on the
ClipboardHelper.SetClipboardData(ocrResult.Text);
ClipboardHelper.SetClipboardData(ocrInformation.Text);
}
exportInformation.ExportMade = true;
}

8
GreenshotWin10Plugin/Processors/Win10OcrProcessor.cs
View file

@ -21,10 +21,8 @@
using System.Threading.Tasks;
using GreenshotPlugin.Core;
using GreenshotPlugin.IniFile;
using GreenshotPlugin.Interfaces;
using GreenshotPlugin.Interfaces.Ocr;
using log4net;
namespace GreenshotWin10Plugin.Processors {
/// <summary>
@ -37,12 +35,18 @@ namespace GreenshotWin10Plugin.Processors {
public override bool ProcessCapture(ISurface surface, ICaptureDetails captureDetails)
{
if (captureDetails.OcrInformation != null)
{
return false;
}
var ocrProvider = SimpleServiceProvider.Current.GetInstance<IOcrProvider>();
var ocrResult = Task.Run(async () => await ocrProvider.DoOcrAsync(surface)).Result;
if (!ocrResult.HasContent) return false;
captureDetails.OcrInformation = ocrResult;
return true;
}
}

11
GreenshotWin10Plugin/Win10OcrProvider.cs
View file

@ -57,13 +57,18 @@ namespace GreenshotWin10Plugin
/// </summary>
/// <param name="surface">ISurface</param>
/// <returns>OcrResult sync</returns>
public Task<OcrInformation> DoOcrAsync(ISurface surface)
public async Task<OcrInformation> DoOcrAsync(ISurface surface)
{
OcrInformation result;
using (var imageStream = new MemoryStream())
{
using var imageStream = new MemoryStream();
ImageOutput.SaveToStream(surface, imageStream, new SurfaceOutputSettings());
imageStream.Position = 0;
var randomAccessStream = imageStream.AsRandomAccessStream();
return DoOcrAsync(randomAccessStream);
result = await DoOcrAsync(randomAccessStream);
}
return result;
}
/// <summary>