Fix Movie Title Slugs #233

src/NzbDrone.Core/Datastore/Migration/117_fix_movie_slugs.cs

@@ -0,0 +1,72 @@
+using System.Data;
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(117)]
+    public class fix_movie_slugs : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Execute.WithConnection(SetTitleSlug);
+        }
+
+        private void SetTitleSlug(IDbConnection conn, IDbTransaction tran)
+        {
+            using (IDbCommand getSeriesCmd = conn.CreateCommand())
+            {
+                getSeriesCmd.Transaction = tran;
+                getSeriesCmd.CommandText = @"SELECT Id, Title, Year FROM Movies";
+                using (IDataReader seriesReader = getSeriesCmd.ExecuteReader())
+                {
+                    while (seriesReader.Read())
+                    {
+                        var id = seriesReader.GetInt32(0);
+                        var title = seriesReader.GetString(1);
+                        var year = seriesReader.GetString(2);
+
+                        var titleSlug = ToUrlSlug(title + "-" + year);
+
+                        using (IDbCommand updateCmd = conn.CreateCommand())
+                        {
+                            updateCmd.Transaction = tran;
+                            updateCmd.CommandText = "UPDATE Movies SET TitleSlug = ? WHERE Id = ?";
+                            updateCmd.AddParameter(titleSlug);
+                            updateCmd.AddParameter(id);
+
+                            updateCmd.ExecuteNonQuery();
+                        }
+                    }
+                }
+            }
+        }
+
+        public static string ToUrlSlug(string value)
+        {
+
+            //First to lower case
+            value = value.ToLowerInvariant();
+
+            //Remove all accents
+            var bytes = Encoding.GetEncoding("Cyrillic").GetBytes(value);
+            value = Encoding.ASCII.GetString(bytes);
+
+            //Replace spaces
+            value = Regex.Replace(value, @"\s", "-", RegexOptions.Compiled);
+
+            //Remove invalid chars
+            value = Regex.Replace(value, @"[^a-z0-9\s-_]", "", RegexOptions.Compiled);
+
+            //Trim dashes from end
+            value = value.Trim('-', '_');
+
+            //Replace double occurences of - or _
+            value = Regex.Replace(value, @"([-_]){2,}", "$1", RegexOptions.Compiled);
+
+            return value;
+        }
+    }
+}

src/NzbDrone.Core/MetadataSource/SkyHook/Resource/TMDBResources.cs

@@ -34,6 +34,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource
         public int vote_count { get; set; }
         public bool video { get; set; }
         public float vote_average { get; set; }
+        public string Slug { get; set; }
     }
 
 

src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs

@@ -13,6 +13,7 @@ using NzbDrone.Core.MetadataSource;
 using NzbDrone.Core.Tv;
 using Newtonsoft.Json;
 using System.Text.RegularExpressions;
+using System.Text;
 
 namespace NzbDrone.Core.MetadataSource.SkyHook
 {
@@ -29,7 +30,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
         public SkyHookProxy(IHttpClient httpClient, ISonarrCloudRequestBuilder requestBuilder, ITmdbConfigService configService, IMovieService movieService, Logger logger)
         {
             _httpClient = httpClient;
-             _requestBuilder = requestBuilder.SkyHookTvdb;
+            _requestBuilder = requestBuilder.SkyHookTvdb;
             _movieBuilder = requestBuilder.TMDB;
             _configService = configService;
             _movieService = movieService;
@@ -88,7 +89,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             movie.TmdbId = TmdbId;
             movie.ImdbId = resource.imdb_id;
             movie.Title = resource.title;
-            movie.TitleSlug = movie.Title.ToLower().Replace(" ", "-");
+            movie.TitleSlug = ToUrlSlug(movie.Title);
             movie.CleanTitle = Parser.Parser.CleanSeriesTitle(movie.Title);
             movie.SortTitle = Parser.Parser.NormalizeTitle(movie.Title);
             movie.Overview = resource.overview;
@@ -104,20 +105,21 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             {
                 _logger.Debug("Movie with this title slug already exists. Adding year...");
             }
+            //movie.TitleSlug += "-" + movie.Year.ToString();
             movie.TitleSlug += "-" + movie.Year.ToString();
 
             movie.Images.Add(_configService.GetCoverForURL(resource.poster_path, MediaCoverTypes.Poster));//TODO: Update to load image specs from tmdb page!
             movie.Images.Add(_configService.GetCoverForURL(resource.backdrop_path, MediaCoverTypes.Banner));
             movie.Runtime = resource.runtime;
 
-            foreach(Title title in resource.alternative_titles.titles)
+            foreach (Title title in resource.alternative_titles.titles)
             {
                 movie.AlternativeTitles.Add(title.title);
             }
 
-            foreach(ReleaseDates releaseDates in resource.release_dates.results)
+            foreach (ReleaseDates releaseDates in resource.release_dates.results)
             {
-                foreach(ReleaseDate releaseDate in releaseDates.release_dates)
+                foreach (ReleaseDate releaseDate in releaseDates.release_dates)
                 {
                     if (releaseDate.type == 5 || releaseDate.type == 4)
                     {
@@ -140,7 +142,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             movie.Ratings.Votes = resource.vote_count;
             movie.Ratings.Value = (decimal)resource.vote_average;
 
-            foreach(Genre genre in resource.genres)
+            foreach (Genre genre in resource.genres)
             {
                 movie.Genres.Add(genre.name);
             }
@@ -176,10 +178,11 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
 
         private string StripTrailingTheFromTitle(string title)
         {
-            if(title.EndsWith(",the"))
+            if (title.EndsWith(",the"))
             {
                 title = title.Substring(0, title.Length - 4);
-            } else if(title.EndsWith(", the"))
+            }
+            else if (title.EndsWith(", the"))
             {
                 title = title.Substring(0, title.Length - 5);
             }
@@ -330,8 +333,8 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             {
                 imdbMovie.SortTitle = Parser.Parser.NormalizeTitle(result.title);
                 imdbMovie.Title = result.title;
-                string titleSlug = result.title;
+                string titleSlug = ToUrlSlug(result.title);
-                imdbMovie.TitleSlug = titleSlug.ToLower().Replace(" ", "-");
+                // imdbMovie.TitleSlug = titleSlug.ToLower().Replace(" ", "-");
 
                 if (result.release_date.IsNotNullOrWhiteSpace())
                 {
@@ -340,12 +343,16 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
 
 
 
-                var slugResult = _movieService.FindByTitleSlug(imdbMovie.TitleSlug);
+                //var slugResult = _movieService.FindByTitleSlug(titleSlug);
-                if (slugResult != null)
+                //if (slugResult != null)
-                {
+                //{
-                    _logger.Debug("Movie with this title slug already exists. Adding year...");
+                //    _logger.Debug("Movie with this title slug already exists. Adding year...");
-                }
+                //}
-                imdbMovie.TitleSlug += "-" + imdbMovie.Year.ToString();
+                //imdbMovie.TitleSlug += "-" + imdbMovie.Year.ToString();
+
+                titleSlug += "-" + imdbMovie.Year.ToString(); ;
+
+                imdbMovie.TitleSlug = titleSlug;
 
                 imdbMovie.Images = new List<MediaCover.MediaCover>();
                 imdbMovie.Overview = result.overview;
@@ -525,5 +532,30 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
                     return MediaCoverTypes.Unknown;
             }
         }
+
+        public static string ToUrlSlug(string value)
+        {
+
+            //First to lower case
+            value = value.ToLowerInvariant();
+
+            //Remove all accents
+            var bytes = Encoding.GetEncoding("Cyrillic").GetBytes(value);
+            value = Encoding.ASCII.GetString(bytes);
+
+            //Replace spaces
+            value = Regex.Replace(value, @"\s", "-", RegexOptions.Compiled);
+
+            //Remove invalid chars
+            value = Regex.Replace(value, @"[^a-z0-9\s-_]", "", RegexOptions.Compiled);
+
+            //Trim dashes from end
+            value = value.Trim('-', '_');
+
+            //Replace double occurences of - or _
+            value = Regex.Replace(value, @"([-_]){2,}", "$1", RegexOptions.Compiled);
+
+            return value;
+        }
     }
 }

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -294,6 +294,7 @@
     </Compile>
     <Compile Include="Datastore\Migration\110_add_physical_release_to_table.cs" />
     <Compile Include="Datastore\Migration\109_add_movie_formats_to_naming_config.cs" />
+    <Compile Include="Datastore\Migration\117_fix_movie_slugs.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationDbFactory.cs" />