From 2cc72e32ad376de2a86cf3c735ada9c9368684cc Mon Sep 17 00:00:00 2001 From: Robin Krom Date: Fri, 28 Oct 2022 00:04:55 +0200 Subject: [PATCH] Created a custom task for generating emoji data, although this works in general, the next build doesn't. --- src/Greenshot.BuildTasks/EmojiDataTask.cs | 178 +++++++++++++++ .../Greenshot.BuildTasks.csproj | 9 + .../Controls/Emoji/EmojiData.cs | 206 ++---------------- src/Greenshot.Editor/Controls/Emoji/Emojis.cs | 15 +- src/Greenshot.Editor/Greenshot.Editor.csproj | 9 - src/Greenshot.sln | 29 ++- src/Greenshot/Greenshot.csproj | 6 + 7 files changed, 243 insertions(+), 209 deletions(-) create mode 100644 src/Greenshot.BuildTasks/EmojiDataTask.cs create mode 100644 src/Greenshot.BuildTasks/Greenshot.BuildTasks.csproj diff --git a/src/Greenshot.BuildTasks/EmojiDataTask.cs b/src/Greenshot.BuildTasks/EmojiDataTask.cs new file mode 100644 index 000000000..f046c6559 --- /dev/null +++ b/src/Greenshot.BuildTasks/EmojiDataTask.cs @@ -0,0 +1,178 @@ +/* + * Greenshot - a free and open source screenshot tool + * Copyright (C) 2007-2021 Thomas Braun, Jens Klingen, Robin Krom + * + * For more information see: https://getgreenshot.org/ + * The Greenshot project is hosted on GitHub https://github.com/greenshot/greenshot + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 1 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Greenshot.Editor.Controls.Emoji; +using SixLabors.Fonts.Unicode; +using Task = Microsoft.Build.Utilities.Task; +using System.Text; +using System.Xml; +using System.Xml.Serialization; +using Microsoft.Build.Framework; + +namespace Greenshot.BuildTasks; + +/// +/// A custom task to generate the emoji data we need for the picker. +/// This is based upon code from Sam Hocevar, the license is here: +/// Emoji.Wpf — Emoji support for WPF +/// +/// Copyright © 2017—2021 Sam Hocevar +/// +/// This library is free software. It comes without any warranty, to +/// the extent permitted by applicable law. You can redistribute it +/// and/or modify it under the terms of the Do What the Fuck You Want +/// to Public License, Version 2, as published by the WTFPL Task Force. +/// See http://www.wtfpl.net/ for more details. +/// +public class EmojiDataTask : Task +{ + private static readonly Regex MatchGroup = new(@"^# group: (.*)", RegexOptions.Compiled); + private static readonly Regex MatchSubgroup = new(@"^# subgroup: (.*)", RegexOptions.Compiled); + private static readonly Regex MatchSequence = new(@"^([0-9a-fA-F ]+[0-9a-fA-F]).*; *([-a-z]*) *# [^ ]* (E[0-9.]* )?(.*)", RegexOptions.Compiled); + + private static string ToColonSyntax(string s) => Regex.Replace(s.Trim().ToLowerInvariant(), "[^a-z0-9]+", "-"); + + /// + /// The name of the output file + /// + [Required] + public string OutputFilename { get; set; } + + //The name of the namespace where the class is going to be generated + [Required] + public string EmojiTestTxtFile { get; set; } + + public override bool Execute() + { + var data = ParseEmojiList(EmojiTestTxtFile); + if (!data.Groups.Any()) + { + return false; + } + Log.LogMessage($"Creating file {OutputFilename}"); + var x = new XmlSerializer(typeof(Emojis)); + using var writer = new XmlTextWriter(OutputFilename, Encoding.UTF8); + x.Serialize(writer, data); + + return true; + } + + + private static Emojis ParseEmojiList(string emojiTestTxtFile) + { + var result = new Emojis(); + var lookupByName = new Dictionary(); + var qualifiedLut = new Dictionary(); + Emojis.Group currentGroup = null; + Emojis.Group currentSubgroup = null; + + foreach (var line in ReadLines(emojiTestTxtFile)) + { + var m = MatchGroup.Match(line); + if (m.Success) + { + currentGroup = new Emojis.Group { Name = m.Groups[1].ToString() }; + result.Groups.Add(currentGroup); + continue; + } + + m = MatchSubgroup.Match(line); + if (m.Success) + { + currentSubgroup = new Emojis.Group { Name = m.Groups[1].ToString() }; + currentGroup?.SubGroups?.Add(currentSubgroup); + continue; + } + + m = MatchSequence.Match(line); + if (!m.Success) + { + continue; + } + string sequence = m.Groups[1].ToString(); + string name = m.Groups[4].ToString(); + + string text = string.Join("", sequence.Split(' ').Select(c => char.ConvertFromUtf32(Convert.ToInt32(c, 16)))); + + // If there is already a differently-qualified version of this character, skip it. + // FIXME: this only works well if fully-qualified appears first in the list. + var unqualified = text.Replace("\ufe0f", ""); + if (qualifiedLut.ContainsKey(unqualified)) + { + continue; + } + + // Fix simple fully-qualified emojis + if (CodePoint.GetCodePointCount(text.AsSpan()) == 2) + { + text = text.TrimEnd('\ufe0f'); + } + + qualifiedLut[unqualified] = text; + + var emoji = new Emojis.Emoji { Text = text }; + + lookupByName[ToColonSyntax(name)] = emoji; + + // Get the left part of the name and check whether we’re a variation of an existing + // emoji. If so, append to that emoji. Otherwise, add to current subgroup. + // FIXME: does not work properly because variations can appear before the generic emoji + if (name.Contains(":") && lookupByName.TryGetValue(ToColonSyntax(name.Split(':')[0]), out var parentEmoji)) + { + parentEmoji.Variations.Add(emoji); + } + else + { + currentSubgroup?.Emojis?.Add(emoji); + } + } + + // Remove the Component group. Not sure we want to have the skin tones in the picker. + result.Groups.RemoveAll(g => g.Name == "Component"); + return result; + } + + /// + /// This reads the specified file into lines + /// + /// string + /// + /// + private static IEnumerable ReadLines(string file) + { + if (!File.Exists(file)) + { + throw new FileNotFoundException($"Can't find {file}"); + } + + using var stream = new FileStream(file, FileMode.Open, FileAccess.Read); + using var reader = new StreamReader(stream, Encoding.UTF8); + while (reader.ReadLine() is { } line) + { + yield return line; + } + } +} \ No newline at end of file diff --git a/src/Greenshot.BuildTasks/Greenshot.BuildTasks.csproj b/src/Greenshot.BuildTasks/Greenshot.BuildTasks.csproj new file mode 100644 index 000000000..9fa6da494 --- /dev/null +++ b/src/Greenshot.BuildTasks/Greenshot.BuildTasks.csproj @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/Greenshot.Editor/Controls/Emoji/EmojiData.cs b/src/Greenshot.Editor/Controls/Emoji/EmojiData.cs index a1772b871..b54bea4bb 100644 --- a/src/Greenshot.Editor/Controls/Emoji/EmojiData.cs +++ b/src/Greenshot.Editor/Controls/Emoji/EmojiData.cs @@ -1,67 +1,37 @@ -// -// Emoji.Wpf — Emoji support for WPF -// -// Copyright © 2017—2021 Sam Hocevar -// -// This library is free software. It comes without any warranty, to -// the extent permitted by applicable law. You can redistribute it -// and/or modify it under the terms of the Do What the Fuck You Want -// to Public License, Version 2, as published by the WTFPL Task Force. -// See http://www.wtfpl.net/ for more details. -// +/* + * Greenshot - a free and open source screenshot tool + * Copyright (C) 2007-2021 Thomas Braun, Jens Klingen, Robin Krom + * + * For more information see: https://getgreenshot.org/ + * The Greenshot project is hosted on GitHub https://github.com/greenshot/greenshot + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 1 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ using System; using System.IO; -using System.Text.RegularExpressions; using System.Xml; using System.Xml.Serialization; -#if DEBUG -using System.Collections.Generic; -using System.IO.Compression; -using System.Linq; -using System.Reflection; -using System.Text; -using SixLabors.Fonts.Unicode; -#endif namespace Greenshot.Editor.Controls.Emoji { /// - /// This class processes the emoji-test.txt to generate a list of possible emoji depending ony skin tone and hairstyle. + /// This class processes the emoji-test.txt extract, as was generated in a build task, so it can show a list of possible emoji depending on skin tone and hairstyle. /// public static class EmojiData { private const string EmojisXmlFilePath = "emojis.xml"; - private const string EmojisTestFile = @"emoji-test.txt.gz"; -#if DEBUG - private const string Adult = "(👨|👩)(🏻|🏼|🏽|🏾|🏿)?"; - private const string Child = "(👦|👧|👶)(🏻|🏼|🏽|🏾|🏿)?"; - private static readonly Regex MatchFamily = new($"{Adult}(\u200d{Adult})*(\u200d{Child})+"); - - private static readonly Regex MatchGroup = new(@"^# group: (.*)", RegexOptions.Compiled); - private static readonly Regex MatchSubgroup = new(@"^# subgroup: (.*)", RegexOptions.Compiled); - private static readonly Regex MatchSequence = new(@"^([0-9a-fA-F ]+[0-9a-fA-F]).*; *([-a-z]*) *# [^ ]* (E[0-9.]* )?(.*)", RegexOptions.Compiled); - private static readonly List SkinToneComponents = new() - { - "🏻", // light skin tone - "🏼", // medium-light skin tone - "🏽", // medium skin tone - "🏾", // medium-dark skin tone - "🏿", // dark skin tone - }; - - private static readonly List HairStyleComponents = new() - { - "🦰", // red hair - "🦱", // curly hair - "🦳", // white hair - "🦲", // bald - }; - - private static readonly Regex MatchSkinTone = new($"({string.Join("|", SkinToneComponents)})", RegexOptions.Compiled); - private static readonly Regex MatchHairStyle = new($"({string.Join("|", HairStyleComponents)})", RegexOptions.Compiled); - -#endif public static Emojis Data { get; private set; } = new(); @@ -73,144 +43,10 @@ namespace Greenshot.Editor.Controls.Emoji { Data = (Emojis)x.Deserialize(new XmlTextReader(EmojisXmlFilePath)); } -#if RELEASE else { throw new NotSupportedException($"Missing {EmojisXmlFilePath}, can't load "); } -#elif DEBUG - else - { - // To be removed - ParseEmojiList(); - if (Data.Groups.Any()) - { - x.Serialize(new XmlTextWriter(EmojisXmlFilePath, Encoding.UTF8), Data); - } - } -#endif } - - -#if DEBUG - private static string ToColonSyntax(string s) => Regex.Replace(s.Trim().ToLowerInvariant(), "[^a-z0-9]+", "-"); - - private static void ParseEmojiList() - { - var lookupByName = new Dictionary(); - - var qualifiedLut = new Dictionary(); - var allText = new List(); - - Emojis.Group currentGroup = null; - Emojis.Group currentSubgroup = null; - - foreach (var line in EmojiDescriptionLines()) - { - var m = MatchGroup.Match(line); - if (m.Success) - { - currentGroup = new Emojis.Group { Name = m.Groups[1].ToString() }; - Data.Groups.Add(currentGroup); - continue; - } - - m = MatchSubgroup.Match(line); - if (m.Success) - { - currentSubgroup = new Emojis.Group { Name = m.Groups[1].ToString() }; - currentGroup?.SubGroups?.Add(currentSubgroup); - continue; - } - - m = MatchSequence.Match(line); - if (!m.Success) - { - continue; - } - string sequence = m.Groups[1].ToString(); - string name = m.Groups[4].ToString(); - - string text = string.Join("", sequence.Split(' ').Select(c => char.ConvertFromUtf32(Convert.ToInt32(c, 16)))); - bool hasModifier = false; - - // If this is a family emoji, no need to add it to our big matching - // regex, since the match_family regex is already included. - if (!MatchFamily.Match(text).Success) - { - // Construct a regex to replace e.g. "🏻" with "(🏻|🏼|🏽|🏾|🏿)" in a big - // regex so that we can match all variations of this Emoji even if they are - // not in the standard. - bool hasNonfirstModifier = false; - var regexText = MatchSkinTone.Replace( - MatchHairStyle.Replace(text, (x) => - { - hasModifier = true; - hasNonfirstModifier |= x.Value != HairStyleComponents[0]; - return MatchHairStyle.ToString(); - }), (x) => - { - hasModifier = true; - hasNonfirstModifier |= x.Value != SkinToneComponents[0]; - return MatchSkinTone.ToString(); - }); - - if (!hasNonfirstModifier) - { - allText.Add(hasModifier ? regexText : text); - } - } - - // If there is already a differently-qualified version of this character, skip it. - // FIXME: this only works well if fully-qualified appears first in the list. - var unqualified = text.Replace("\ufe0f", ""); - if (qualifiedLut.ContainsKey(unqualified)) - { - continue; - } - - // Fix simple fully-qualified emojis - if (CodePoint.GetCodePointCount(text.AsSpan()) == 2) - { - text = text.TrimEnd('\ufe0f'); - } - - qualifiedLut[unqualified] = text; - - var emoji = new Emojis.Emoji { Text = text}; - - lookupByName[ToColonSyntax(name)] = emoji; - - // Get the left part of the name and check whether we’re a variation of an existing - // emoji. If so, append to that emoji. Otherwise, add to current subgroup. - // FIXME: does not work properly because variations can appear before the generic emoji - if (name.Contains(":") && lookupByName.TryGetValue(ToColonSyntax(name.Split(':')[0]), out var parentEmoji)) - { - parentEmoji.Variations.Add(emoji); - } - else - { - currentSubgroup?.Emojis?.Add(emoji); - } - } - - // Remove the Component group. Not sure we want to have the skin tones in the picker. - Data.Groups.RemoveAll(g => g.Name == "Component"); - } - - private static IEnumerable EmojiDescriptionLines() - { - var exeDirectory = Path.GetDirectoryName(Assembly.GetCallingAssembly().Location); - var emojiTestFile = Path.Combine(exeDirectory, EmojisTestFile); - if (!File.Exists(emojiTestFile)) - { - throw new FileNotFoundException($"Can't find {emojiTestFile}, bad installation?"); - } - using var fileStream = new FileStream(emojiTestFile, FileMode.Open, FileAccess.Read); - using var gzStream = new GZipStream(fileStream, CompressionMode.Decompress); - using var streamReader = new StreamReader(gzStream); - return streamReader.ReadToEnd().Split('\r', '\n'); - } -#endif } } \ No newline at end of file diff --git a/src/Greenshot.Editor/Controls/Emoji/Emojis.cs b/src/Greenshot.Editor/Controls/Emoji/Emojis.cs index 039dfaaf8..4314718b0 100644 --- a/src/Greenshot.Editor/Controls/Emoji/Emojis.cs +++ b/src/Greenshot.Editor/Controls/Emoji/Emojis.cs @@ -26,20 +26,22 @@ using System.Xml.Serialization; namespace Greenshot.Editor.Controls.Emoji; +[XmlRoot("Es")] public class Emojis { - [XmlElement(ElementName = "Group")] + [XmlArray(ElementName = "Gs")] public List Groups { get; set; } = new(); + [XmlType("G")] public class Group { - [XmlAttribute] + [XmlAttribute(AttributeName= "N")] public string Name { get; set; } - [XmlElement(ElementName = "Group")] + [XmlArray(ElementName = "Sg")] public List SubGroups { get; set; } = new(); - [XmlElement(ElementName = "Emoji")] + [XmlArray(ElementName = "Es")] public List Emojis { get; set; } = new(); public IEnumerable> EmojiChunkList => new ChunkHelper(EmojiList, 8); @@ -49,12 +51,13 @@ public class Emojis public IEnumerable EmojiList => SubGroups.SelectMany(s => s.Emojis); } + [XmlType("E")] public class Emoji { - [XmlAttribute] + [XmlAttribute(AttributeName = "T")] public string Text { get; set; } - [XmlArray] + [XmlArray(ElementName = "V")] public List Variations { get; set; } = new(); /// diff --git a/src/Greenshot.Editor/Greenshot.Editor.csproj b/src/Greenshot.Editor/Greenshot.Editor.csproj index b4c743cb5..37825ea1e 100644 --- a/src/Greenshot.Editor/Greenshot.Editor.csproj +++ b/src/Greenshot.Editor/Greenshot.Editor.csproj @@ -15,11 +15,6 @@ - - - - @@ -99,10 +94,6 @@ - - PreserveNewest - emoji-test.txt.gz - PreserveNewest TwemojiMozilla.ttf.gz diff --git a/src/Greenshot.sln b/src/Greenshot.sln index 91ddf4314..cdb9f6565 100644 --- a/src/Greenshot.sln +++ b/src/Greenshot.sln @@ -1,21 +1,22 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29728.190 +# Visual Studio Version 17 +VisualStudioVersion = 17.3.32929.385 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Greenshot", "Greenshot\Greenshot.csproj", "{CD642BF4-D815-4D67-A0B5-C69F0B8231AF}" ProjectSection(ProjectDependencies) = postProject - {92599C09-FF29-4ABD-B6E6-C48ECD781BAB} = {92599C09-FF29-4ABD-B6E6-C48ECD781BAB} + {1893A2E4-A78A-4713-A8E7-E70058DABEE0} = {1893A2E4-A78A-4713-A8E7-E70058DABEE0} {19FEEF09-313F-43C7-819D-F1BCA782B08B} = {19FEEF09-313F-43C7-819D-F1BCA782B08B} + {25C870BE-22BB-4CB8-B274-95DC481F53A7} = {25C870BE-22BB-4CB8-B274-95DC481F53A7} + {47F23C86-604E-4CC3-8767-B3D4088F30BB} = {47F23C86-604E-4CC3-8767-B3D4088F30BB} + {697CF066-9077-4F22-99D9-D989CCE7282B} = {697CF066-9077-4F22-99D9-D989CCE7282B} + {7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E} = {7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E} + {80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50} = {80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50} + {92599C09-FF29-4ABD-B6E6-C48ECD781BAB} = {92599C09-FF29-4ABD-B6E6-C48ECD781BAB} {9801F62C-540F-4BFE-9211-6405DEDE563B} = {9801F62C-540F-4BFE-9211-6405DEDE563B} {9C0ECC4C-7807-4111-916A-4F57BB29788A} = {9C0ECC4C-7807-4111-916A-4F57BB29788A} - {C3052651-598A-44E2-AAB3-2E41311D50F9} = {C3052651-598A-44E2-AAB3-2E41311D50F9} - {7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E} = {7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E} - {697CF066-9077-4F22-99D9-D989CCE7282B} = {697CF066-9077-4F22-99D9-D989CCE7282B} - {47F23C86-604E-4CC3-8767-B3D4088F30BB} = {47F23C86-604E-4CC3-8767-B3D4088F30BB} - {80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50} = {80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50} {AD7CFFE2-40E7-46CF-A172-D48CF7AE9A12} = {AD7CFFE2-40E7-46CF-A172-D48CF7AE9A12} - {1893A2E4-A78A-4713-A8E7-E70058DABEE0} = {1893A2E4-A78A-4713-A8E7-E70058DABEE0} + {C3052651-598A-44E2-AAB3-2E41311D50F9} = {C3052651-598A-44E2-AAB3-2E41311D50F9} EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Greenshot.Base", "Greenshot.Base\Greenshot.Base.csproj", "{5B924697-4DCD-4F98-85F1-105CB84B7341}" @@ -52,6 +53,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Greenshot.Editor", "Greenshot.Editor\Greenshot.Editor.csproj", "{148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Greenshot.BuildTasks", "Greenshot.BuildTasks\Greenshot.BuildTasks.csproj", "{25C870BE-22BB-4CB8-B274-95DC481F53A7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -172,6 +175,14 @@ Global {148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}.Release|Any CPU.Build.0 = Release|Any CPU {148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}.Release|x86.ActiveCfg = Release|Any CPU {148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}.Release|x86.Build.0 = Release|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|x86.ActiveCfg = Debug|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|x86.Build.0 = Debug|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|Any CPU.Build.0 = Release|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|x86.ActiveCfg = Release|Any CPU + {25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Greenshot/Greenshot.csproj b/src/Greenshot/Greenshot.csproj index fc4dd90d1..5d7356e18 100644 --- a/src/Greenshot/Greenshot.csproj +++ b/src/Greenshot/Greenshot.csproj @@ -65,6 +65,12 @@ + + + + + +