Created a custom task for generating emoji data, although this works in general, the next build doesn't.

This commit is contained in:
Robin Krom 2022-10-28 00:04:55 +02:00
commit 2cc72e32ad
No known key found for this signature in database
GPG key ID: BCC01364F1371490
7 changed files with 243 additions and 209 deletions

View file

@ -0,0 +1,178 @@
/*
* Greenshot - a free and open source screenshot tool
* Copyright (C) 2007-2021 Thomas Braun, Jens Klingen, Robin Krom
*
* For more information see: https://getgreenshot.org/
* The Greenshot project is hosted on GitHub https://github.com/greenshot/greenshot
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Greenshot.Editor.Controls.Emoji;
using SixLabors.Fonts.Unicode;
using Task = Microsoft.Build.Utilities.Task;
using System.Text;
using System.Xml;
using System.Xml.Serialization;
using Microsoft.Build.Framework;
namespace Greenshot.BuildTasks;
/// <summary>
/// A custom task to generate the emoji data we need for the picker.
/// This is based upon code from Sam Hocevar, the license is here:
/// Emoji.Wpf — Emoji support for WPF
///
/// Copyright © 2017—2021 Sam Hocevar <sam@hocevar.net>
///
/// This library is free software. It comes without any warranty, to
/// the extent permitted by applicable law. You can redistribute it
/// and/or modify it under the terms of the Do What the Fuck You Want
/// to Public License, Version 2, as published by the WTFPL Task Force.
/// See http://www.wtfpl.net/ for more details.
/// </summary>
public class EmojiDataTask : Task
{
private static readonly Regex MatchGroup = new(@"^# group: (.*)", RegexOptions.Compiled);
private static readonly Regex MatchSubgroup = new(@"^# subgroup: (.*)", RegexOptions.Compiled);
private static readonly Regex MatchSequence = new(@"^([0-9a-fA-F ]+[0-9a-fA-F]).*; *([-a-z]*) *# [^ ]* (E[0-9.]* )?(.*)", RegexOptions.Compiled);
private static string ToColonSyntax(string s) => Regex.Replace(s.Trim().ToLowerInvariant(), "[^a-z0-9]+", "-");
/// <summary>
/// The name of the output file
/// </summary>
[Required]
public string OutputFilename { get; set; }
//The name of the namespace where the class is going to be generated
[Required]
public string EmojiTestTxtFile { get; set; }
public override bool Execute()
{
var data = ParseEmojiList(EmojiTestTxtFile);
if (!data.Groups.Any())
{
return false;
}
Log.LogMessage($"Creating file {OutputFilename}");
var x = new XmlSerializer(typeof(Emojis));
using var writer = new XmlTextWriter(OutputFilename, Encoding.UTF8);
x.Serialize(writer, data);
return true;
}
private static Emojis ParseEmojiList(string emojiTestTxtFile)
{
var result = new Emojis();
var lookupByName = new Dictionary<string, Emojis.Emoji>();
var qualifiedLut = new Dictionary<string, string>();
Emojis.Group currentGroup = null;
Emojis.Group currentSubgroup = null;
foreach (var line in ReadLines(emojiTestTxtFile))
{
var m = MatchGroup.Match(line);
if (m.Success)
{
currentGroup = new Emojis.Group { Name = m.Groups[1].ToString() };
result.Groups.Add(currentGroup);
continue;
}
m = MatchSubgroup.Match(line);
if (m.Success)
{
currentSubgroup = new Emojis.Group { Name = m.Groups[1].ToString() };
currentGroup?.SubGroups?.Add(currentSubgroup);
continue;
}
m = MatchSequence.Match(line);
if (!m.Success)
{
continue;
}
string sequence = m.Groups[1].ToString();
string name = m.Groups[4].ToString();
string text = string.Join("", sequence.Split(' ').Select(c => char.ConvertFromUtf32(Convert.ToInt32(c, 16))));
// If there is already a differently-qualified version of this character, skip it.
// FIXME: this only works well if fully-qualified appears first in the list.
var unqualified = text.Replace("\ufe0f", "");
if (qualifiedLut.ContainsKey(unqualified))
{
continue;
}
// Fix simple fully-qualified emojis
if (CodePoint.GetCodePointCount(text.AsSpan()) == 2)
{
text = text.TrimEnd('\ufe0f');
}
qualifiedLut[unqualified] = text;
var emoji = new Emojis.Emoji { Text = text };
lookupByName[ToColonSyntax(name)] = emoji;
// Get the left part of the name and check whether were a variation of an existing
// emoji. If so, append to that emoji. Otherwise, add to current subgroup.
// FIXME: does not work properly because variations can appear before the generic emoji
if (name.Contains(":") && lookupByName.TryGetValue(ToColonSyntax(name.Split(':')[0]), out var parentEmoji))
{
parentEmoji.Variations.Add(emoji);
}
else
{
currentSubgroup?.Emojis?.Add(emoji);
}
}
// Remove the Component group. Not sure we want to have the skin tones in the picker.
result.Groups.RemoveAll(g => g.Name == "Component");
return result;
}
/// <summary>
/// This reads the specified file into lines
/// </summary>
/// <param name="file">string</param>
/// <returns></returns>
/// <exception cref="FileNotFoundException"></exception>
private static IEnumerable<string> ReadLines(string file)
{
if (!File.Exists(file))
{
throw new FileNotFoundException($"Can't find {file}");
}
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
using var reader = new StreamReader(stream, Encoding.UTF8);
while (reader.ReadLine() is { } line)
{
yield return line;
}
}
}

View file

@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">
<ItemGroup>
<PackageReference Include="Microsoft.Build.Utilities.Core" version="17.3.2" />
<PackageReference Include="SixLabors.ImageSharp.Drawing" version="1.0.0-beta15" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Greenshot.Editor\Greenshot.Editor.csproj" />
</ItemGroup>
</Project>

View file

@ -1,67 +1,37 @@
//
// Emoji.Wpf — Emoji support for WPF
//
// Copyright © 2017—2021 Sam Hocevar <sam@hocevar.net>
//
// This library is free software. It comes without any warranty, to
// the extent permitted by applicable law. You can redistribute it
// and/or modify it under the terms of the Do What the Fuck You Want
// to Public License, Version 2, as published by the WTFPL Task Force.
// See http://www.wtfpl.net/ for more details.
//
/*
* Greenshot - a free and open source screenshot tool
* Copyright (C) 2007-2021 Thomas Braun, Jens Klingen, Robin Krom
*
* For more information see: https://getgreenshot.org/
* The Greenshot project is hosted on GitHub https://github.com/greenshot/greenshot
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
using System;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Serialization;
#if DEBUG
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Text;
using SixLabors.Fonts.Unicode;
#endif
namespace Greenshot.Editor.Controls.Emoji
{
/// <summary>
/// This class processes the emoji-test.txt to generate a list of possible emoji depending ony skin tone and hairstyle.
/// This class processes the emoji-test.txt extract, as was generated in a build task, so it can show a list of possible emoji depending on skin tone and hairstyle.
/// </summary>
public static class EmojiData
{
private const string EmojisXmlFilePath = "emojis.xml";
private const string EmojisTestFile = @"emoji-test.txt.gz";
#if DEBUG
private const string Adult = "(👨|👩)(🏻|🏼|🏽|🏾|🏿)?";
private const string Child = "(👦|👧|👶)(🏻|🏼|🏽|🏾|🏿)?";
private static readonly Regex MatchFamily = new($"{Adult}(\u200d{Adult})*(\u200d{Child})+");
private static readonly Regex MatchGroup = new(@"^# group: (.*)", RegexOptions.Compiled);
private static readonly Regex MatchSubgroup = new(@"^# subgroup: (.*)", RegexOptions.Compiled);
private static readonly Regex MatchSequence = new(@"^([0-9a-fA-F ]+[0-9a-fA-F]).*; *([-a-z]*) *# [^ ]* (E[0-9.]* )?(.*)", RegexOptions.Compiled);
private static readonly List<string> SkinToneComponents = new()
{
"🏻", // light skin tone
"🏼", // medium-light skin tone
"🏽", // medium skin tone
"🏾", // medium-dark skin tone
"🏿", // dark skin tone
};
private static readonly List<string> HairStyleComponents = new()
{
"🦰", // red hair
"🦱", // curly hair
"🦳", // white hair
"🦲", // bald
};
private static readonly Regex MatchSkinTone = new($"({string.Join("|", SkinToneComponents)})", RegexOptions.Compiled);
private static readonly Regex MatchHairStyle = new($"({string.Join("|", HairStyleComponents)})", RegexOptions.Compiled);
#endif
public static Emojis Data { get; private set; } = new();
@ -73,144 +43,10 @@ namespace Greenshot.Editor.Controls.Emoji
{
Data = (Emojis)x.Deserialize(new XmlTextReader(EmojisXmlFilePath));
}
#if RELEASE
else
{
throw new NotSupportedException($"Missing {EmojisXmlFilePath}, can't load ");
}
#elif DEBUG
else
{
// To be removed
ParseEmojiList();
if (Data.Groups.Any())
{
x.Serialize(new XmlTextWriter(EmojisXmlFilePath, Encoding.UTF8), Data);
}
}
#endif
}
#if DEBUG
private static string ToColonSyntax(string s) => Regex.Replace(s.Trim().ToLowerInvariant(), "[^a-z0-9]+", "-");
private static void ParseEmojiList()
{
var lookupByName = new Dictionary<string, Emojis.Emoji>();
var qualifiedLut = new Dictionary<string, string>();
var allText = new List<string>();
Emojis.Group currentGroup = null;
Emojis.Group currentSubgroup = null;
foreach (var line in EmojiDescriptionLines())
{
var m = MatchGroup.Match(line);
if (m.Success)
{
currentGroup = new Emojis.Group { Name = m.Groups[1].ToString() };
Data.Groups.Add(currentGroup);
continue;
}
m = MatchSubgroup.Match(line);
if (m.Success)
{
currentSubgroup = new Emojis.Group { Name = m.Groups[1].ToString() };
currentGroup?.SubGroups?.Add(currentSubgroup);
continue;
}
m = MatchSequence.Match(line);
if (!m.Success)
{
continue;
}
string sequence = m.Groups[1].ToString();
string name = m.Groups[4].ToString();
string text = string.Join("", sequence.Split(' ').Select(c => char.ConvertFromUtf32(Convert.ToInt32(c, 16))));
bool hasModifier = false;
// If this is a family emoji, no need to add it to our big matching
// regex, since the match_family regex is already included.
if (!MatchFamily.Match(text).Success)
{
// Construct a regex to replace e.g. "🏻" with "(🏻|🏼|🏽|🏾|🏿)" in a big
// regex so that we can match all variations of this Emoji even if they are
// not in the standard.
bool hasNonfirstModifier = false;
var regexText = MatchSkinTone.Replace(
MatchHairStyle.Replace(text, (x) =>
{
hasModifier = true;
hasNonfirstModifier |= x.Value != HairStyleComponents[0];
return MatchHairStyle.ToString();
}), (x) =>
{
hasModifier = true;
hasNonfirstModifier |= x.Value != SkinToneComponents[0];
return MatchSkinTone.ToString();
});
if (!hasNonfirstModifier)
{
allText.Add(hasModifier ? regexText : text);
}
}
// If there is already a differently-qualified version of this character, skip it.
// FIXME: this only works well if fully-qualified appears first in the list.
var unqualified = text.Replace("\ufe0f", "");
if (qualifiedLut.ContainsKey(unqualified))
{
continue;
}
// Fix simple fully-qualified emojis
if (CodePoint.GetCodePointCount(text.AsSpan()) == 2)
{
text = text.TrimEnd('\ufe0f');
}
qualifiedLut[unqualified] = text;
var emoji = new Emojis.Emoji { Text = text};
lookupByName[ToColonSyntax(name)] = emoji;
// Get the left part of the name and check whether were a variation of an existing
// emoji. If so, append to that emoji. Otherwise, add to current subgroup.
// FIXME: does not work properly because variations can appear before the generic emoji
if (name.Contains(":") && lookupByName.TryGetValue(ToColonSyntax(name.Split(':')[0]), out var parentEmoji))
{
parentEmoji.Variations.Add(emoji);
}
else
{
currentSubgroup?.Emojis?.Add(emoji);
}
}
// Remove the Component group. Not sure we want to have the skin tones in the picker.
Data.Groups.RemoveAll(g => g.Name == "Component");
}
private static IEnumerable<string> EmojiDescriptionLines()
{
var exeDirectory = Path.GetDirectoryName(Assembly.GetCallingAssembly().Location);
var emojiTestFile = Path.Combine(exeDirectory, EmojisTestFile);
if (!File.Exists(emojiTestFile))
{
throw new FileNotFoundException($"Can't find {emojiTestFile}, bad installation?");
}
using var fileStream = new FileStream(emojiTestFile, FileMode.Open, FileAccess.Read);
using var gzStream = new GZipStream(fileStream, CompressionMode.Decompress);
using var streamReader = new StreamReader(gzStream);
return streamReader.ReadToEnd().Split('\r', '\n');
}
#endif
}
}

View file

@ -26,20 +26,22 @@ using System.Xml.Serialization;
namespace Greenshot.Editor.Controls.Emoji;
[XmlRoot("Es")]
public class Emojis
{
[XmlElement(ElementName = "Group")]
[XmlArray(ElementName = "Gs")]
public List<Group> Groups { get; set; } = new();
[XmlType("G")]
public class Group
{
[XmlAttribute]
[XmlAttribute(AttributeName= "N")]
public string Name { get; set; }
[XmlElement(ElementName = "Group")]
[XmlArray(ElementName = "Sg")]
public List<Group> SubGroups { get; set; } = new();
[XmlElement(ElementName = "Emoji")]
[XmlArray(ElementName = "Es")]
public List<Emoji> Emojis { get; set; } = new();
public IEnumerable<IEnumerable<Emoji>> EmojiChunkList => new ChunkHelper<Emoji>(EmojiList, 8);
@ -49,12 +51,13 @@ public class Emojis
public IEnumerable<Emoji> EmojiList => SubGroups.SelectMany(s => s.Emojis);
}
[XmlType("E")]
public class Emoji
{
[XmlAttribute]
[XmlAttribute(AttributeName = "T")]
public string Text { get; set; }
[XmlArray]
[XmlArray(ElementName = "V")]
public List<Emoji> Variations { get; set; } = new();
/// <summary>

View file

@ -15,11 +15,6 @@
<ProjectReference Include="..\Greenshot.Base\Greenshot.Base.csproj" />
</ItemGroup>
<Target Name="CompressEmojiTxtResource" BeforeTargets="BeforeBuild" Condition="!Exists('Controls\Emoji\emoji-test.txt.gz')">
<GZipTask InputFile="Controls\Emoji\emoji-test.txt"
OutputFile="Controls\Emoji\emoji-test.txt.gz" />
</Target>
<Target Name="CompressTwemojiMozillaTtfResource" BeforeTargets="BeforeBuild" Condition="!Exists('Drawing\Emoji\TwemojiMozilla.ttf.gz')">
<GZipTask InputFile="Drawing\Emoji\TwemojiMozilla.ttf"
OutputFile="Drawing\Emoji\TwemojiMozilla.ttf.gz" />
@ -99,10 +94,6 @@
</ItemGroup>
<ItemGroup>
<ContentWithTargetPath Include="Controls\Emoji\emoji-test.txt.gz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<TargetPath>emoji-test.txt.gz</TargetPath>
</ContentWithTargetPath>
<ContentWithTargetPath Include="Drawing\Emoji\TwemojiMozilla.ttf.gz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<TargetPath>TwemojiMozilla.ttf.gz</TargetPath>

View file

@ -1,21 +1,22 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29728.190
# Visual Studio Version 17
VisualStudioVersion = 17.3.32929.385
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Greenshot", "Greenshot\Greenshot.csproj", "{CD642BF4-D815-4D67-A0B5-C69F0B8231AF}"
ProjectSection(ProjectDependencies) = postProject
{92599C09-FF29-4ABD-B6E6-C48ECD781BAB} = {92599C09-FF29-4ABD-B6E6-C48ECD781BAB}
{1893A2E4-A78A-4713-A8E7-E70058DABEE0} = {1893A2E4-A78A-4713-A8E7-E70058DABEE0}
{19FEEF09-313F-43C7-819D-F1BCA782B08B} = {19FEEF09-313F-43C7-819D-F1BCA782B08B}
{25C870BE-22BB-4CB8-B274-95DC481F53A7} = {25C870BE-22BB-4CB8-B274-95DC481F53A7}
{47F23C86-604E-4CC3-8767-B3D4088F30BB} = {47F23C86-604E-4CC3-8767-B3D4088F30BB}
{697CF066-9077-4F22-99D9-D989CCE7282B} = {697CF066-9077-4F22-99D9-D989CCE7282B}
{7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E} = {7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E}
{80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50} = {80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50}
{92599C09-FF29-4ABD-B6E6-C48ECD781BAB} = {92599C09-FF29-4ABD-B6E6-C48ECD781BAB}
{9801F62C-540F-4BFE-9211-6405DEDE563B} = {9801F62C-540F-4BFE-9211-6405DEDE563B}
{9C0ECC4C-7807-4111-916A-4F57BB29788A} = {9C0ECC4C-7807-4111-916A-4F57BB29788A}
{C3052651-598A-44E2-AAB3-2E41311D50F9} = {C3052651-598A-44E2-AAB3-2E41311D50F9}
{7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E} = {7EC72A5A-D73A-4B4B-9CA1-2216C7D92D5E}
{697CF066-9077-4F22-99D9-D989CCE7282B} = {697CF066-9077-4F22-99D9-D989CCE7282B}
{47F23C86-604E-4CC3-8767-B3D4088F30BB} = {47F23C86-604E-4CC3-8767-B3D4088F30BB}
{80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50} = {80D8DEB9-94E3-4876-8CCA-2DF1ED5F2C50}
{AD7CFFE2-40E7-46CF-A172-D48CF7AE9A12} = {AD7CFFE2-40E7-46CF-A172-D48CF7AE9A12}
{1893A2E4-A78A-4713-A8E7-E70058DABEE0} = {1893A2E4-A78A-4713-A8E7-E70058DABEE0}
{C3052651-598A-44E2-AAB3-2E41311D50F9} = {C3052651-598A-44E2-AAB3-2E41311D50F9}
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Greenshot.Base", "Greenshot.Base\Greenshot.Base.csproj", "{5B924697-4DCD-4F98-85F1-105CB84B7341}"
@ -52,6 +53,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Greenshot.Editor", "Greenshot.Editor\Greenshot.Editor.csproj", "{148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Greenshot.BuildTasks", "Greenshot.BuildTasks\Greenshot.BuildTasks.csproj", "{25C870BE-22BB-4CB8-B274-95DC481F53A7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -172,6 +175,14 @@ Global
{148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}.Release|Any CPU.Build.0 = Release|Any CPU
{148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}.Release|x86.ActiveCfg = Release|Any CPU
{148D3C8B-D6EC-4A7D-80E9-243A81F19DD2}.Release|x86.Build.0 = Release|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|x86.ActiveCfg = Debug|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Debug|x86.Build.0 = Debug|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|Any CPU.Build.0 = Release|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|x86.ActiveCfg = Release|Any CPU
{25C870BE-22BB-4CB8-B274-95DC481F53A7}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

View file

@ -65,6 +65,12 @@
</Task>
</UsingTask>
<UsingTask TaskName="EmojiDataTask" AssemblyFile="..\Greenshot.BuildTasks\bin\Debug\net472\Greenshot.BuildTasks.dll" />
<Target Name="GenerateEmojiTestTxtFile" BeforeTargets="BeforeBuild">
<EmojiDataTask EmojiTestTxtFile="..\Greenshot.Editor\Controls\Emoji\emoji-test.txt" OutputFilename="$(SolutionDir)$(SolutionName)\$(OutDir)\emojis.xml" />
</Target>
<Target Name="Generate hashes" BeforeTargets="PostBuildEvent">
<GetFileHash Files="@(FilesToHash)" Algorithm="SHA256" HashEncoding="hex">
<Output TaskParameter="Items" ItemName="FilesWithHashes" />