Add metadata export function

This commit is contained in:
JonnyWong16 2020-08-03 10:35:17 -07:00
parent 0ff363b6ee
commit c102020698
No known key found for this signature in database
GPG key ID: B1F1F9807184697A
3 changed files with 972 additions and 0 deletions

916
plexpy/exporter.py Normal file
View file

@ -0,0 +1,916 @@
# -*- coding: utf-8 -*-
# This file is part of Tautulli.
#
# Tautulli is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Tautulli is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Tautulli. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import csv
import json
import os
from functools import partial
from io import open
from multiprocessing.dummy import Pool as ThreadPool
import plexpy
if plexpy.PYTHON2:
import helpers
import logger
from plex import Plex
else:
from plexpy import helpers
from plexpy import logger
from plexpy.plex import Plex
MOVIE_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'audienceRating': None,
'audienceRatingImage': None,
'chapters': {
'id': None,
'tag': None,
'index': None,
'start': None,
'end': None,
'thumb': None
},
'chapterSource': None,
'collections': {
'id': None,
'tag': None
},
'contentRating': None,
'countries': {
'id': None,
'tag': None
},
'directors': {
'id': None,
'tag': None
},
'duration': None,
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'key': None,
'labels': {
'id': None,
'tag': None
},
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'media': {
'aspectRatio': None,
'audioChannels': None,
'audioCodec': None,
'audioProfile': None,
'bitrate': None,
'container': None,
'duration': None,
'height': None,
'id': None,
'has64bitOffsets': None,
'optimizedForStreaming': None,
'optimizedVersion': None,
'target': None,
'title': None,
'videoCodec': None,
'videoFrameRate': None,
'videoProfile': None,
'videoResolution': None,
'width': None,
'parts': {
'accessible': None,
'audioProfile': None,
'container': None,
'deepAnalysisVersion': None,
'duration': None,
'exists': None,
'file': None,
'has64bitOffsets': None,
'id': None,
'indexes': None,
'key': None,
'size': None,
'optimizedForStreaming': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'syncItemId': None,
'syncState': None,
'videoProfile': None,
'videoStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'bitDepth': None,
'bitrate': None,
'cabac': None,
'chromaLocation': None,
'chromaSubsampling': None,
'colorPrimaries': None,
'colorRange': None,
'colorSpace': None,
'colorTrc': None,
'duration': None,
'frameRate': None,
'frameRateMode': None,
'hasScalingMatrix': None,
'height': None,
'level': None,
'profile': None,
'refFrames': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'scanType': None,
'streamIdentifier': None,
'width': None
},
'audioStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'audioChannelLayout': None,
'bitDepth': None,
'bitrate': None,
'bitrateMode': None,
'channels': None,
'dialogNorm': None,
'duration': None,
'profile': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'samplingRate': None
},
'subtitleStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'forced': None,
'format': None,
'key': None
}
}
},
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'originalTitle': None,
'producers': {
'id': None,
'tag': None
},
'rating': None,
'ratingImage': None,
'ratingKey': None,
'roles': {
'id': None,
'tag': None,
'role': None,
'thumb': None
},
'studio': None,
'summary': None,
'tagline': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'writers': {
'id': None,
'tag': None
},
'year': None
}
SHOW_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'banner': None,
'childCount': None,
'collections': {
'id': None,
'tag': None
},
'contentRating': None,
'duration': None,
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'index': None,
'key': None,
'labels': {
'id': None,
'tag': None
},
'lastViewedAt': helpers.datetime_to_iso,
'leafCount': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'rating': None,
'ratingKey': None,
'roles': {
'id': None,
'tag': None,
'role': None,
'thumb': None
},
'studio': None,
'summary': None,
'theme': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'viewedLeafCount': None,
'year': None,
'seasons': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
SEASON_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'fields': {
'name': None,
'locked': None
},
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'leafCount': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentTheme': None,
'parentThumb': None,
'parentTitle': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'viewedLeafCount': None,
'episodes': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
EPISODE_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'chapterSource': None,
'contentRating': None,
'directors': {
'id': None,
'tag': None
},
'duration': None,
'fields': {
'name': None,
'locked': None
},
'grandparentArt': None,
'grandparentGuid': None,
'grandparentKey': None,
'grandparentRatingKey': None,
'grandparentTheme': None,
'grandparentThumb': None,
'grandparentTitle': None,
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'media': {
'aspectRatio': None,
'audioChannels': None,
'audioCodec': None,
'audioProfile': None,
'bitrate': None,
'container': None,
'duration': None,
'height': None,
'id': None,
'has64bitOffsets': None,
'optimizedForStreaming': None,
'optimizedVersion': None,
'target': None,
'title': None,
'videoCodec': None,
'videoFrameRate': None,
'videoProfile': None,
'videoResolution': None,
'width': None,
'parts': {
'accessible': None,
'audioProfile': None,
'container': None,
'deepAnalysisVersion': None,
'duration': None,
'exists': None,
'file': None,
'has64bitOffsets': None,
'id': None,
'indexes': None,
'key': None,
'size': None,
'optimizedForStreaming': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'syncItemId': None,
'syncState': None,
'videoProfile': None,
'videoStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'bitDepth': None,
'bitrate': None,
'cabac': None,
'chromaLocation': None,
'chromaSubsampling': None,
'colorPrimaries': None,
'colorRange': None,
'colorSpace': None,
'colorTrc': None,
'duration': None,
'frameRate': None,
'frameRateMode': None,
'hasScalingMatrix': None,
'height': None,
'level': None,
'profile': None,
'refFrames': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'scanType': None,
'streamIdentifier': None,
'width': None
},
'audioStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'audioChannelLayout': None,
'bitDepth': None,
'bitrate': None,
'bitrateMode': None,
'channels': None,
'dialogNorm': None,
'duration': None,
'profile': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'samplingRate': None
},
'subtitleStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'forced': None,
'format': None,
'key': None
}
}
},
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'rating': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'writers': {
'id': None,
'tag': None
},
'year': None
}
ARTIST_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'collections': {
'id': None,
'tag': None
},
'countries': {
'id': None,
'tag': None
},
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'moods': {
'id': None,
'tag': None
},
'rating': None,
'ratingKey': None,
'styles': {
'id': None,
'tag': None
},
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'albums': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
ALBUM_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'collections': {
'id': None,
'tag': None
},
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'index': None,
'key': None,
'labels': {
'id': None,
'tag': None
},
'lastViewedAt': helpers.datetime_to_iso,
'leafCount': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'loudnessAnalysisVersion': None,
'moods': {
'id': None,
'tag': None
},
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'parentGuid': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'rating': None,
'ratingKey': None,
'styles': {
'id': None,
'tag': None
},
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'viewedLeafCount': None,
'tracks': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
TRACK_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'duration': None,
'grandparentArt': None,
'grandparentGuid': None,
'grandparentKey': None,
'grandparentRatingKey': None,
'grandparentThumb': None,
'grandparentTitle': None,
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'media': {
'audioChannels': None,
'audioCodec': None,
'audioProfile': None,
'bitrate': None,
'container': None,
'duration': None,
'id': None,
'title': None,
'parts': {
'accessible': None,
'audioProfile': None,
'container': None,
'deepAnalysisVersion': None,
'duration': None,
'exists': None,
'file': None,
'hasThumbnail': None,
'id': None,
'key': None,
'size': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'syncItemId': None,
'syncState': None,
'audioStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'albumGain': None,
'albumPeak': None,
'albumRange': None,
'audioChannelLayout': None,
'bitrate': None,
'channels': None,
'duration': None,
'endRamp': None,
'gain': None,
'loudness': None,
'lra': None,
'peak': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'samplingRate': None,
'startRamp': None,
},
'lyricStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'minLines': None,
'provider': None,
'streamType': None,
'timed': None,
'title': None,
'type': None,
'format': None,
'key': None
}
}
},
'moods': {
'id': None,
'tag': None
},
'originalTitle': None,
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'ratingCount': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'year': None,
}
PHOTO_ALBUM_ATTRS = {
# For some reason photos needs to be first,
# otherwise the photo album ratingKey gets
# clobbered by the first photo's ratingKey
'photos': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type]),
'addedAt': helpers.datetime_to_iso,
'art': None,
'composite': None,
'guid': None,
'index': None,
'key': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso
}
PHOTO_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'createdAtAccuracy': None,
'createdAtTZOffset': None,
'guid': None,
'index': None,
'key': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'year': None,
'media': {
'aperture': None,
'aspectRatio': None,
'container': None,
'height': None,
'id': None,
'iso': None,
'lens': None,
'make': None,
'model': None,
'width': None,
'parts': {
'accessible': None,
'container': None,
'exists': None,
'file': None,
'id': None,
'key': None,
'size': None
}
},
'tag': {
'id': None,
'tag': None,
'title': None
}
}
COLLECTION_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'childCount': None,
'collectionMode': None,
'collectionSort': None,
'contentRating': None,
'fields': {
'name': None,
'locked': None
},
'guid': None,
'index': None,
'key': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'maxYear': None,
'minYear': None,
'ratingKey': None,
'subtype': None,
'summary': None,
'thumb': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'children': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
PLAYLIST_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'composite': None,
'duration': None,
'guid': None,
'key': None,
'leafCount': None,
'playlistType': None,
'ratingKey': None,
'smart': None,
'summary': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'items': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
MEDIA_TYPES = {
'movie': MOVIE_ATTRS,
'show': SHOW_ATTRS,
'season': SEASON_ATTRS,
'episode': EPISODE_ATTRS,
'artist': ARTIST_ATTRS,
'album': ALBUM_ATTRS,
'track': TRACK_ATTRS,
'photo album': PHOTO_ALBUM_ATTRS,
'photo': PHOTO_ATTRS,
'collection': COLLECTION_ATTRS,
'playlist': PLAYLIST_ATTRS
}
def export(section_id=None, rating_key=None, output_format='json'):
timestamp = helpers.timestamp()
if not section_id and not rating_key:
logger.error("Tautulli Exporter :: Export called but no section_id or rating_key provided.")
return
elif section_id and not str(section_id).isdigit():
logger.error("Tautulli Exporter :: Export called with invalid section_id '%s'.", section_id)
return
elif rating_key and not str(rating_key).isdigit():
logger.error("Tautulli Exporter :: Export called with invalid rating_key '%s'.", rating_key)
return
elif output_format not in ('json', 'csv'):
logger.error("Tautulli Exporter :: Export called but invalid output_format '%s' provided.", output_format)
return
plex = Plex(plexpy.CONFIG.PMS_URL, plexpy.CONFIG.PMS_TOKEN)
if section_id:
logger.debug("Tautulli Exporter :: Exporting called with section_id %s", section_id)
library = plex.get_library(section_id)
media_type = library.type
library_title = library.title
filename = 'Library - {} [{}].{}.{}'.format(
library_title, section_id, helpers.timestamp_to_YMDHMS(timestamp), output_format)
items = library.all()
elif rating_key:
logger.debug("Tautulli Exporter :: Exporting called with rating_key %s", rating_key)
item = plex.get_item(helpers.cast_to_int(rating_key))
media_type = item.type
if media_type in ('season', 'episode', 'album', 'track'):
item_title = item._defaultSyncTitle()
else:
item_title = item.title
if media_type == 'photo' and item.TAG == 'Directory':
media_type = 'photo album'
filename = '{} - {} [{}].{}.{}'.format(
media_type.title(), item_title, rating_key, helpers.timestamp_to_YMDHMS(timestamp), output_format)
items = [item]
else:
return
filename = helpers.clean_filename(filename)
filepath = os.path.join(plexpy.CONFIG.CACHE_DIR, filename)
logger.info("Tautulli Exporter :: Starting export for '%s'...", filename)
attrs = MEDIA_TYPES[media_type]
part = partial(helpers.get_attrs_to_dict, attrs=attrs)
with ThreadPool(processes=4) as pool:
result = pool.map(part, items)
if output_format == 'json':
with open(filepath, 'w', encoding='utf-8') as outfile:
json.dump(result, outfile, indent=4, ensure_ascii=False, sort_keys=True)
elif output_format == 'csv':
flatten_result = helpers.flatten_dict(result)
flatten_attrs = helpers.flatten_dict(attrs)
with open(filepath, 'w', encoding='utf-8', newline='') as outfile:
writer = csv.DictWriter(outfile, sorted(flatten_attrs[0].keys()))
writer.writeheader()
writer.writerows(flatten_result)
logger.info("Tautulli Exporter :: Successfully exported to '%s'", filepath)

42
plexpy/plex.py Normal file
View file

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
# This file is part of Tautulli.
#
# Tautulli is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Tautulli is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Tautulli. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from future.builtins import object
from future.builtins import str
from plexapi.server import PlexServer
import plexpy
if plexpy.PYTHON2:
import logger
else:
from plexpy import logger
class Plex(object):
def __init__(self, url, token):
self.plex = PlexServer(url, token)
def get_library(self, section_id):
return self.plex.library.sectionByID(str(section_id))
def get_library_items(self, section_id):
return self.get_library(str(section_id)).all()
def get_item(self, rating_key):
return self.plex.fetchItem(rating_key)

View file

@ -48,6 +48,7 @@ if plexpy.PYTHON2:
import config
import database
import datafactory
import exporter
import graphs
import helpers
import http_handler
@ -81,6 +82,7 @@ else:
from plexpy import config
from plexpy import database
from plexpy import datafactory
from plexpy import exporter
from plexpy import graphs
from plexpy import helpers
from plexpy import http_handler
@ -6414,3 +6416,15 @@ class WebInterface(object):
status['message'] = 'Database not ok'
return status
@cherrypy.expose
@cherrypy.tools.json_out()
@requireAuth(member_of("admin"))
@addtoapi()
def export_metadata(self, section_id=None, rating_key=None, output_format='json', **kwargs):
threading.Thread(target=exporter.export,
kwargs={'section_id': section_id,
'rating_key': rating_key,
'output_format': output_format}).start()
return {'result': 'success',
'message': 'Metadata export has started. Check the logs to monitor any problems.'}