Add metadata export function

This commit is contained in:
JonnyWong16 2020-08-03 10:35:17 -07:00
parent 0ff363b6ee
commit c102020698
No known key found for this signature in database
GPG key ID: B1F1F9807184697A
3 changed files with 972 additions and 0 deletions

916
plexpy/exporter.py Normal file
View file

@ -0,0 +1,916 @@
# -*- coding: utf-8 -*-
# This file is part of Tautulli.
#
# Tautulli is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Tautulli is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Tautulli. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
import csv
import json
import os
from functools import partial
from io import open
from multiprocessing.dummy import Pool as ThreadPool
import plexpy
if plexpy.PYTHON2:
import helpers
import logger
from plex import Plex
else:
from plexpy import helpers
from plexpy import logger
from plexpy.plex import Plex
MOVIE_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'audienceRating': None,
'audienceRatingImage': None,
'chapters': {
'id': None,
'tag': None,
'index': None,
'start': None,
'end': None,
'thumb': None
},
'chapterSource': None,
'collections': {
'id': None,
'tag': None
},
'contentRating': None,
'countries': {
'id': None,
'tag': None
},
'directors': {
'id': None,
'tag': None
},
'duration': None,
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'key': None,
'labels': {
'id': None,
'tag': None
},
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'media': {
'aspectRatio': None,
'audioChannels': None,
'audioCodec': None,
'audioProfile': None,
'bitrate': None,
'container': None,
'duration': None,
'height': None,
'id': None,
'has64bitOffsets': None,
'optimizedForStreaming': None,
'optimizedVersion': None,
'target': None,
'title': None,
'videoCodec': None,
'videoFrameRate': None,
'videoProfile': None,
'videoResolution': None,
'width': None,
'parts': {
'accessible': None,
'audioProfile': None,
'container': None,
'deepAnalysisVersion': None,
'duration': None,
'exists': None,
'file': None,
'has64bitOffsets': None,
'id': None,
'indexes': None,
'key': None,
'size': None,
'optimizedForStreaming': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'syncItemId': None,
'syncState': None,
'videoProfile': None,
'videoStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'bitDepth': None,
'bitrate': None,
'cabac': None,
'chromaLocation': None,
'chromaSubsampling': None,
'colorPrimaries': None,
'colorRange': None,
'colorSpace': None,
'colorTrc': None,
'duration': None,
'frameRate': None,
'frameRateMode': None,
'hasScalingMatrix': None,
'height': None,
'level': None,
'profile': None,
'refFrames': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'scanType': None,
'streamIdentifier': None,
'width': None
},
'audioStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'audioChannelLayout': None,
'bitDepth': None,
'bitrate': None,
'bitrateMode': None,
'channels': None,
'dialogNorm': None,
'duration': None,
'profile': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'samplingRate': None
},
'subtitleStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'forced': None,
'format': None,
'key': None
}
}
},
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'originalTitle': None,
'producers': {
'id': None,
'tag': None
},
'rating': None,
'ratingImage': None,
'ratingKey': None,
'roles': {
'id': None,
'tag': None,
'role': None,
'thumb': None
},
'studio': None,
'summary': None,
'tagline': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'writers': {
'id': None,
'tag': None
},
'year': None
}
SHOW_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'banner': None,
'childCount': None,
'collections': {
'id': None,
'tag': None
},
'contentRating': None,
'duration': None,
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'index': None,
'key': None,
'labels': {
'id': None,
'tag': None
},
'lastViewedAt': helpers.datetime_to_iso,
'leafCount': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'rating': None,
'ratingKey': None,
'roles': {
'id': None,
'tag': None,
'role': None,
'thumb': None
},
'studio': None,
'summary': None,
'theme': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'viewedLeafCount': None,
'year': None,
'seasons': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
SEASON_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'fields': {
'name': None,
'locked': None
},
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'leafCount': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentTheme': None,
'parentThumb': None,
'parentTitle': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'viewedLeafCount': None,
'episodes': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
EPISODE_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'chapterSource': None,
'contentRating': None,
'directors': {
'id': None,
'tag': None
},
'duration': None,
'fields': {
'name': None,
'locked': None
},
'grandparentArt': None,
'grandparentGuid': None,
'grandparentKey': None,
'grandparentRatingKey': None,
'grandparentTheme': None,
'grandparentThumb': None,
'grandparentTitle': None,
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'media': {
'aspectRatio': None,
'audioChannels': None,
'audioCodec': None,
'audioProfile': None,
'bitrate': None,
'container': None,
'duration': None,
'height': None,
'id': None,
'has64bitOffsets': None,
'optimizedForStreaming': None,
'optimizedVersion': None,
'target': None,
'title': None,
'videoCodec': None,
'videoFrameRate': None,
'videoProfile': None,
'videoResolution': None,
'width': None,
'parts': {
'accessible': None,
'audioProfile': None,
'container': None,
'deepAnalysisVersion': None,
'duration': None,
'exists': None,
'file': None,
'has64bitOffsets': None,
'id': None,
'indexes': None,
'key': None,
'size': None,
'optimizedForStreaming': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'syncItemId': None,
'syncState': None,
'videoProfile': None,
'videoStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'bitDepth': None,
'bitrate': None,
'cabac': None,
'chromaLocation': None,
'chromaSubsampling': None,
'colorPrimaries': None,
'colorRange': None,
'colorSpace': None,
'colorTrc': None,
'duration': None,
'frameRate': None,
'frameRateMode': None,
'hasScalingMatrix': None,
'height': None,
'level': None,
'profile': None,
'refFrames': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'scanType': None,
'streamIdentifier': None,
'width': None
},
'audioStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'audioChannelLayout': None,
'bitDepth': None,
'bitrate': None,
'bitrateMode': None,
'channels': None,
'dialogNorm': None,
'duration': None,
'profile': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'samplingRate': None
},
'subtitleStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'language': None,
'languageCode': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'forced': None,
'format': None,
'key': None
}
}
},
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'rating': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'writers': {
'id': None,
'tag': None
},
'year': None
}
ARTIST_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'collections': {
'id': None,
'tag': None
},
'countries': {
'id': None,
'tag': None
},
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'locations': None,
'moods': {
'id': None,
'tag': None
},
'rating': None,
'ratingKey': None,
'styles': {
'id': None,
'tag': None
},
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'albums': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
ALBUM_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'collections': {
'id': None,
'tag': None
},
'fields': {
'name': None,
'locked': None
},
'genres': {
'id': None,
'tag': None
},
'guid': None,
'index': None,
'key': None,
'labels': {
'id': None,
'tag': None
},
'lastViewedAt': helpers.datetime_to_iso,
'leafCount': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'loudnessAnalysisVersion': None,
'moods': {
'id': None,
'tag': None
},
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'parentGuid': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'rating': None,
'ratingKey': None,
'styles': {
'id': None,
'tag': None
},
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'viewedLeafCount': None,
'tracks': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
TRACK_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'art': None,
'duration': None,
'grandparentArt': None,
'grandparentGuid': None,
'grandparentKey': None,
'grandparentRatingKey': None,
'grandparentThumb': None,
'grandparentTitle': None,
'guid': None,
'index': None,
'key': None,
'lastViewedAt': helpers.datetime_to_iso,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'media': {
'audioChannels': None,
'audioCodec': None,
'audioProfile': None,
'bitrate': None,
'container': None,
'duration': None,
'id': None,
'title': None,
'parts': {
'accessible': None,
'audioProfile': None,
'container': None,
'deepAnalysisVersion': None,
'duration': None,
'exists': None,
'file': None,
'hasThumbnail': None,
'id': None,
'key': None,
'size': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'syncItemId': None,
'syncState': None,
'audioStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'selected': None,
'streamType': None,
'title': None,
'type': None,
'albumGain': None,
'albumPeak': None,
'albumRange': None,
'audioChannelLayout': None,
'bitrate': None,
'channels': None,
'duration': None,
'endRamp': None,
'gain': None,
'loudness': None,
'lra': None,
'peak': None,
'requiredBandwidths': lambda e: [int(b) for b in e.split(',')] if e else None,
'samplingRate': None,
'startRamp': None,
},
'lyricStreams': {
'codec': None,
'codecID': None,
'displayTitle': None,
'extendedDisplayTitle': None,
'id': None,
'index': None,
'minLines': None,
'provider': None,
'streamType': None,
'timed': None,
'title': None,
'type': None,
'format': None,
'key': None
}
}
},
'moods': {
'id': None,
'tag': None
},
'originalTitle': None,
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'ratingCount': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'titleSort': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'userRating': None,
'viewCount': None,
'year': None,
}
PHOTO_ALBUM_ATTRS = {
# For some reason photos needs to be first,
# otherwise the photo album ratingKey gets
# clobbered by the first photo's ratingKey
'photos': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type]),
'addedAt': helpers.datetime_to_iso,
'art': None,
'composite': None,
'guid': None,
'index': None,
'key': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso
}
PHOTO_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'createdAtAccuracy': None,
'createdAtTZOffset': None,
'guid': None,
'index': None,
'key': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'originallyAvailableAt': partial(helpers.datetime_to_iso, to_date=True),
'parentGuid': None,
'parentIndex': None,
'parentKey': None,
'parentRatingKey': None,
'parentThumb': None,
'parentTitle': None,
'ratingKey': None,
'summary': None,
'thumb': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'year': None,
'media': {
'aperture': None,
'aspectRatio': None,
'container': None,
'height': None,
'id': None,
'iso': None,
'lens': None,
'make': None,
'model': None,
'width': None,
'parts': {
'accessible': None,
'container': None,
'exists': None,
'file': None,
'id': None,
'key': None,
'size': None
}
},
'tag': {
'id': None,
'tag': None,
'title': None
}
}
COLLECTION_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'childCount': None,
'collectionMode': None,
'collectionSort': None,
'contentRating': None,
'fields': {
'name': None,
'locked': None
},
'guid': None,
'index': None,
'key': None,
'librarySectionID': None,
'librarySectionKey': None,
'librarySectionTitle': None,
'maxYear': None,
'minYear': None,
'ratingKey': None,
'subtype': None,
'summary': None,
'thumb': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'children': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
PLAYLIST_ATTRS = {
'addedAt': helpers.datetime_to_iso,
'composite': None,
'duration': None,
'guid': None,
'key': None,
'leafCount': None,
'playlistType': None,
'ratingKey': None,
'smart': None,
'summary': None,
'title': None,
'type': None,
'updatedAt': helpers.datetime_to_iso,
'items': lambda e: helpers.get_attrs_to_dict(e, MEDIA_TYPES[e.type])
}
MEDIA_TYPES = {
'movie': MOVIE_ATTRS,
'show': SHOW_ATTRS,
'season': SEASON_ATTRS,
'episode': EPISODE_ATTRS,
'artist': ARTIST_ATTRS,
'album': ALBUM_ATTRS,
'track': TRACK_ATTRS,
'photo album': PHOTO_ALBUM_ATTRS,
'photo': PHOTO_ATTRS,
'collection': COLLECTION_ATTRS,
'playlist': PLAYLIST_ATTRS
}
def export(section_id=None, rating_key=None, output_format='json'):
timestamp = helpers.timestamp()
if not section_id and not rating_key:
logger.error("Tautulli Exporter :: Export called but no section_id or rating_key provided.")
return
elif section_id and not str(section_id).isdigit():
logger.error("Tautulli Exporter :: Export called with invalid section_id '%s'.", section_id)
return
elif rating_key and not str(rating_key).isdigit():
logger.error("Tautulli Exporter :: Export called with invalid rating_key '%s'.", rating_key)
return
elif output_format not in ('json', 'csv'):
logger.error("Tautulli Exporter :: Export called but invalid output_format '%s' provided.", output_format)
return
plex = Plex(plexpy.CONFIG.PMS_URL, plexpy.CONFIG.PMS_TOKEN)
if section_id:
logger.debug("Tautulli Exporter :: Exporting called with section_id %s", section_id)
library = plex.get_library(section_id)
media_type = library.type
library_title = library.title
filename = 'Library - {} [{}].{}.{}'.format(
library_title, section_id, helpers.timestamp_to_YMDHMS(timestamp), output_format)
items = library.all()
elif rating_key:
logger.debug("Tautulli Exporter :: Exporting called with rating_key %s", rating_key)
item = plex.get_item(helpers.cast_to_int(rating_key))
media_type = item.type
if media_type in ('season', 'episode', 'album', 'track'):
item_title = item._defaultSyncTitle()
else:
item_title = item.title
if media_type == 'photo' and item.TAG == 'Directory':
media_type = 'photo album'
filename = '{} - {} [{}].{}.{}'.format(
media_type.title(), item_title, rating_key, helpers.timestamp_to_YMDHMS(timestamp), output_format)
items = [item]
else:
return
filename = helpers.clean_filename(filename)
filepath = os.path.join(plexpy.CONFIG.CACHE_DIR, filename)
logger.info("Tautulli Exporter :: Starting export for '%s'...", filename)
attrs = MEDIA_TYPES[media_type]
part = partial(helpers.get_attrs_to_dict, attrs=attrs)
with ThreadPool(processes=4) as pool:
result = pool.map(part, items)
if output_format == 'json':
with open(filepath, 'w', encoding='utf-8') as outfile:
json.dump(result, outfile, indent=4, ensure_ascii=False, sort_keys=True)
elif output_format == 'csv':
flatten_result = helpers.flatten_dict(result)
flatten_attrs = helpers.flatten_dict(attrs)
with open(filepath, 'w', encoding='utf-8', newline='') as outfile:
writer = csv.DictWriter(outfile, sorted(flatten_attrs[0].keys()))
writer.writeheader()
writer.writerows(flatten_result)
logger.info("Tautulli Exporter :: Successfully exported to '%s'", filepath)

42
plexpy/plex.py Normal file
View file

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
# This file is part of Tautulli.
#
# Tautulli is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Tautulli is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Tautulli. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from future.builtins import object
from future.builtins import str
from plexapi.server import PlexServer
import plexpy
if plexpy.PYTHON2:
import logger
else:
from plexpy import logger
class Plex(object):
def __init__(self, url, token):
self.plex = PlexServer(url, token)
def get_library(self, section_id):
return self.plex.library.sectionByID(str(section_id))
def get_library_items(self, section_id):
return self.get_library(str(section_id)).all()
def get_item(self, rating_key):
return self.plex.fetchItem(rating_key)

View file

@ -48,6 +48,7 @@ if plexpy.PYTHON2:
import config import config
import database import database
import datafactory import datafactory
import exporter
import graphs import graphs
import helpers import helpers
import http_handler import http_handler
@ -81,6 +82,7 @@ else:
from plexpy import config from plexpy import config
from plexpy import database from plexpy import database
from plexpy import datafactory from plexpy import datafactory
from plexpy import exporter
from plexpy import graphs from plexpy import graphs
from plexpy import helpers from plexpy import helpers
from plexpy import http_handler from plexpy import http_handler
@ -6414,3 +6416,15 @@ class WebInterface(object):
status['message'] = 'Database not ok' status['message'] = 'Database not ok'
return status return status
@cherrypy.expose
@cherrypy.tools.json_out()
@requireAuth(member_of("admin"))
@addtoapi()
def export_metadata(self, section_id=None, rating_key=None, output_format='json', **kwargs):
threading.Thread(target=exporter.export,
kwargs={'section_id': section_id,
'rating_key': rating_key,
'output_format': output_format}).start()
return {'result': 'success',
'message': 'Metadata export has started. Check the logs to monitor any problems.'}