From 4ea9b06e71bd6d7f073f6205080591bd1f6e1171 Mon Sep 17 00:00:00 2001 From: Janne Koschinski <janne@kuschku.de> Date: Wed, 29 Apr 2020 20:06:35 +0200 Subject: [PATCH] Implement episode metadata handling --- src/api/imdb_api.js | 22 +++++++ src/directory_walker.js | 34 ++++++++-- src/index.js | 52 ++++++++++----- src/metadata_loader.js | 138 ++++++++++++++++++++++++++++++---------- src/util/video-mime.js | 6 +- 5 files changed, 197 insertions(+), 55 deletions(-) diff --git a/src/api/imdb_api.js b/src/api/imdb_api.js index 12313c7..db88097 100644 --- a/src/api/imdb_api.js +++ b/src/api/imdb_api.js @@ -29,6 +29,14 @@ class ImdbApi { }); } + findEpisodeById(id, seasonNumber, episodeNumber) { + return this.queryJson(ImdbApi.queryGetEpisode, { + 1: id, + 2: seasonNumber, + 3: episodeNumber, + }); + } + search(type, title, year) { return this.query(ImdbApi.querySearch, { 1: type, @@ -119,6 +127,20 @@ class ImdbApi { LEFT OUTER JOIN title_crew on title.tconst = title_crew.tconst WHERE title.tconst = ? `; + + static queryGetEpisode = ` + SELECT json_object( + 'id', title.tconst, + 'primaryTitle', title.primaryTitle, + 'originalTitle', title.originalTitle, + 'runtimeMinutes', title.runtimeMinutes + ) AS json + FROM title_episode + JOIN title ON title_episode.tconst = title.tconst + WHERE title_episode.parentTconst = ? + AND seasonNumber = ? + AND episodeNumber = ? + `; } function promisify(db, fun) { diff --git a/src/directory_walker.js b/src/directory_walker.js index 59a4071..e840f60 100644 --- a/src/directory_walker.js +++ b/src/directory_walker.js @@ -46,19 +46,43 @@ class FileManager { .then(result => result .filter(dir => dir.isDirectory()) .map(dir => { - return path.join(this.moviesPath, dir.name) + return path.join(this.showsPath, dir.name) }) ); } + async listEpisodes(showPath) { + return fsPromises + .readdir(showPath, {withFileTypes: true}) + .then(result => result + .filter(dir => dir.isDirectory()) + .map(dir => { + const match = /^(?:(?:(?<year>\d+)-(?<month>\d+)-(?<day>\d+))|(?:(?:S(?<season>\d+(?:[.\-–—&]\d+)?))?\p{L}*[\t\f ]*(?<episode>\d+(?:[.\-–—&]\d+)?)))(?:(?:[\t\f ]+(?:[\-–—:][\t\f ]*)?)(?<title>\S.*))?$/u.exec(dir.name); + if (!match) { + return null; + } + const {season, episode, year, month, day} = match.groups + return { + filePath: path.join(showPath, dir.name), + src: encodePath(path.relative(this.basePath, path.join(showPath, dir.name))), + episodeIdentifier: { + season: season, + episode: episode, + year: year, + month: month, + day: day, + }, + } + }).filter(el => el !== null) + ); + } + async findIds(filePath) { return await fsPromises - .readFile(path.join(filePath, "metadata.json")) + .readFile(path.join(filePath, "ids.json")) .then(result => { if (!result) return null; - const json = JSON.parse(result) - if (!json) return null; - return json.ids + return JSON.parse(result); }).catch(_ => null); } diff --git a/src/index.js b/src/index.js index c69f990..af8e389 100644 --- a/src/index.js +++ b/src/index.js @@ -31,6 +31,10 @@ async function main() { const {name, year} = /^(?<name>.+) \((?<year>\d+)\)$/.exec(path.basename(filePath)).groups; const ids = (await fileManager.findIds(filePath)) || (await loader.identifyMovie(name, year)); + if (!ids) { + console.error(`Could not identify movie ${name} (${year}) at ${filePath}`) + return; + } const [media, {metadata, images}] = await Promise.all([ fileManager.findMedia(filePath), loader.loadMetadata(ids), @@ -52,34 +56,52 @@ async function main() { ]); } - async function processEpisode(ids, episodeIdentifier, filePath) { - const episodeIds = loader.identifyEpisode(ids, episodeIdentifier); - const {episodeMetadata, _} = await loader.loadMetadata(episodeIds, true) + async function processEpisode(showIds, episodeIdentifier, filePath) { + const [media, {metadata, images}] = await Promise.all([ + fileManager.findMedia(filePath), + loader.loadEpisodeMetadata(showIds, episodeIdentifier), + ]); + const imageData = await loader.processImages(basePath, filePath, images); await Promise.all([ - fsPromises.writeFile(path.join(filePath, "ids.json"), JSON.stringify(episodeIds, null, 2)), - fsPromises.writeFile(path.join(filePath, "metadata.json"), JSON.stringify(episodeMetadata, null, 2)), + fsPromises.writeFile(path.join(filePath, "metadata.json"), JSON.stringify({ + ...metadata, + ...media, + images: imageData.map(img => { + return { + type: img.type, + src: img.src, + } + }), + }, null, 2)), ]); } async function processShow(filePath) { - if (1) return; - const {name, year} = /^(?<name>.+) \((?<year>\d+)\)$/.exec(path.basename(filePath)).groups; const ids = (await fileManager.findIds(filePath)) || (await loader.identifyShow(name, year)); - const episodes = fileManager.findEpisodes(filePath); + if (!ids) { + console.error(`Could not identify show ${name} (${year}) at ${filePath}`) + return; + } + const episodes = await fileManager.listEpisodes(filePath); - const {metadata, rawImages} = await Promise.all([ - loader.loadMetadata(ids) - ]); - const images = await loader.processImages(basePath, filePath, rawImages); + const {metadata, images} = await loader.loadMetadata(ids); + const imageData = await loader.processImages(basePath, filePath, images); await Promise.all([ - ...episodes.map(async ({episodeIdentifier, filePath}) => await processEpisode(ids, episodeIdentifier, filePath)), + ...episodes.map(async ({episodeIdentifier, filePath}) => await processEpisode(ids, episodeIdentifier, filePath).catch(err => { + console.error(`Error processing episode ${JSON.stringify(episodeIdentifier)} of show ${JSON.stringify(ids)}: `, err); + })), fsPromises.writeFile(path.join(filePath, "ids.json"), JSON.stringify(ids, null, 2)), fsPromises.writeFile(path.join(filePath, "metadata.json"), JSON.stringify({ ...metadata, - episodes: episodes, - images: images, + episodes: episodes.map(episode => { + return { + src: episode.src, + episodeIdentifier: episode.episodeIdentifier, + } + }), + images: imageData, }, null, 2)), ]); } diff --git a/src/metadata_loader.js b/src/metadata_loader.js index 3357826..9745d9e 100644 --- a/src/metadata_loader.js +++ b/src/metadata_loader.js @@ -61,9 +61,9 @@ class MetadataLoader { }), ratings: tmdbContentRatings.results.map(el => { const certification = - Array.isArray(el) ? el.release_dates.sort((a, b) => b.type - a.type).map(el => el.certification) : - el ? el.rating : - null; + Array.isArray(el.release_dates) ? el.release_dates.sort((a, b) => b.type - a.type).map(el => el.certification)[0] : + el ? el.rating : + null; return { region: el.iso_3166_1, certification: certification, @@ -72,6 +72,37 @@ class MetadataLoader { } } + transformEpisodeData(ids, imdbResult, tmdbResult, tmdbTranslations) { + if (!imdbResult) return null; + if (!tmdbResult) return null; + if (!tmdbTranslations) return null; + + return { + ids: ids, + originalLanguage: tmdbResult.original_language, + originalTitle: imdbResult.originalTitle, + primaryTitle: imdbResult.primaryTitle, + titles: tmdbTranslations.translations.map(el => { + return { + title: el.data.name, + region: el.iso_3166_1, + languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], + } + }).filter(el => el.overview), + primaryDescription: { + overview: tmdbResult.overview, + }, + descriptions: tmdbTranslations.translations.map(el => { + return { + region: el.iso_3166_1, + languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], + overview: el.data.overview, + } + }).filter(el => el.overview), + runtime: imdbResult.runtimeMinutes + } + } + chooseImages(originalLanguage, tmdbImages, fanartImages) { function findbest(list) { let sorted = list.sort((a, b) => b.likes - a.likes); @@ -107,9 +138,11 @@ class MetadataLoader { return { logo: fanartImages && fanartImages.hdmovielogo ? findbest(fanartImages.hdmovielogo) : null, - poster: tmdbImages.posters.map(calculateConfidenceAndQuality(true, originalLanguage)) + poster: tmdbImages.posters && tmdbImages.posters.map(calculateConfidenceAndQuality(true, originalLanguage)) .sort(imageComparator)[0], - backdrop: tmdbImages.backdrops.map(calculateConfidenceAndQuality(false, originalLanguage)) + backdrop: tmdbImages.backdrops && tmdbImages.backdrops.map(calculateConfidenceAndQuality(false, originalLanguage)) + .sort(imageComparator)[0], + still: tmdbImages.stills && tmdbImages.stills.map(calculateConfidenceAndQuality(false, originalLanguage)) .sort(imageComparator)[0], } } @@ -119,45 +152,59 @@ class MetadataLoader { query: title, primary_release_year: year }).catch((e) => console.error(e)); + if (!results) return null; const result = results.results.sort((a, b) => { return b.popularity - a.popularity; })[0]; + if (!result) return null; + + const tmdbResult = await this.tmdb.request(`movie/${result.id}`); + if (!tmdbResult) return null; - return result ? { + return { uuid: uuid.v4(), + imdb: tmdbResult.imdb_id, tmdb: result.id, - imdb: (await this.tmdb.request(`movie/${result.id}`)).imdb_id - } : null; + tvdb: null, + } } - async identifyShow(title, year) { - /* - FIXME: Implement this properly, previous implementation for clarity below + async identifyShow(showTitle, showYear) { + const tvdbResults = await this.tvdb.getSeriesByName(showTitle) + if (!tvdbResults) return null; - const imdbId = await this.imdb.search("tvSeries", title, year); + const result = tvdbResults.find(show => { + const {year} = /^(?<year>\d+)(?:-(?<month>\d+)(?:-(?<day>\d+))?)?$/.exec(show.firstAired).groups; + return year === showYear; + }); + if (!result) return null; - const tvdbResults = await this.tvdb.getSeriesByImdbId(imdbId).catch((e) => console.error(e)); - const tvdbResult = tvdbResults[0]; + const tvdbId = result.id; + if (!tvdbId) return null; - if (!tvdbResult) return null; + const tvdbSeries = await this.tvdb.getSeriesById(tvdbId); + if (!tvdbSeries) return null; - const tmdbResults = await this.tmdb.request(`find/${imdbId}`, { - "external_source": "imdb_id", - }).catch((e) => console.error(e)); + const imdbId = tvdbSeries.imdbId; + const tmdbResults = (await this.tmdb.request(`find/${imdbId}`, { + external_source: "imdb_id" + })) || (await this.tmdb.request(`find/${tvdbId}`, { + external_source: "tvdb_id" + })) + if (!tmdbResults) return null; - const tmdbResult = tmdbResults.tv_results.sort((a, b) => { - return b.popularity - a.popularity; - })[0]; + const tmdbSeries = tmdbResults.tv_results ? tmdbResults.tv_results[0] : null; + if (!tmdbSeries) return null; - if (!tmdbResult) return null; + const tmdbId = tmdbSeries.id; return { + uuid: uuid.v4(), imdb: imdbId, - tvdb: tvdbResult.id, - tmdb: tmdbResult.id, - }; - */ + tvdb: tvdbId, + tmdb: tmdbId, + } } async processImages(basePath, filePath, images) { @@ -176,6 +223,11 @@ class MetadataLoader { type: "backdrop", url: this.tmdb.getImageUrl(images.backdrop.file_path), src: encodePath(path.relative(basePath, path.join(filePath, "metadata", `backdrop${path.extname(images.backdrop.file_path)}`))) + }, + !images.still ? null : !images.still.file_path ? null : { + type: "still", + url: this.tmdb.getImageUrl(images.still.file_path), + src: encodePath(path.relative(basePath, path.join(filePath, "metadata", `still${path.extname(images.still.file_path)}`))) } ].filter(el => el !== null); @@ -184,16 +236,37 @@ class MetadataLoader { return imageData; } - async loadMetadata(ids, isEpisode) { + async loadEpisodeMetadata(ids, episodeIdentifier) { + const {season, episode} = episodeIdentifier; + const tmdbSources = [ + `tv/${ids.tmdb}/season/${season}/episode/${episode}`, + `tv/${ids.tmdb}/season/${season}/episode/${episode}/translations`, + `tv/${ids.tmdb}/season/${season}/episode/${episode}/images` + ] + + const [imdbResult, tmdbResult, tmdbTranslations, tmdbImages] = await Promise.all([ + this.imdb.findEpisodeById(ids.imdb, season, episode).catch(_ => null), + ...tmdbSources.map(url => this.tmdb.request(url).catch(_ => null)), + ].filter(el => el !== null)); + + const metadata = this.transformEpisodeData(ids, imdbResult, tmdbResult, tmdbTranslations); + if (!metadata) { + return null; + } + return { + metadata: metadata, + images: this.chooseImages(metadata.originalLanguage, tmdbImages), + }; + } + + async loadMetadata(ids) { const titleType = await this.imdb.findTypeById(ids.imdb); function tmdbSources() { if (titleType !== "tvSeries") { return [`movie/${ids.tmdb}`, `movie/${ids.tmdb}/translations`, `movie/${ids.tmdb}/release_dates`, `movie/${ids.tmdb}/images`] - } else if (!isEpisode) { - return [`tv/${ids.tmdb}`, `tv/${ids.tmdb}/translations`, `tv/${ids.tmdb}/content_ratings`, `tv/${ids.tmdb}/images`] } else { - return [`tv/${ids.tmdb}`, `tv/${ids.tmdb}/translations`] + return [`tv/${ids.tmdb}`, `tv/${ids.tmdb}/translations`, `tv/${ids.tmdb}/content_ratings`, `tv/${ids.tmdb}/images`] } } @@ -204,14 +277,13 @@ class MetadataLoader { const [imdbResult, tmdbResult, tmdbTranslations, tmdbContentRatings, tmdbImages, fanartImages] = await Promise.all([ this.imdb.findById(ids.imdb).catch(_ => null), ...tmdbSources().map(url => this.tmdb.request(url).catch(_ => null)), - isEpisode ? null : this.fanart.request(fanartSource).catch(_ => null) // do nothing, it just means it wasn’t found + this.fanart.request(fanartSource).catch(_ => null) // do nothing, it just means it wasn’t found ].filter(el => el !== null)); const metadata = this.transformData(ids, imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations); return { metadata: metadata, - // also use tvdb for images - images: isEpisode ? null : this.chooseImages(metadata.originalLanguage, tmdbImages, fanartImages), + images: this.chooseImages(metadata.originalLanguage, tmdbImages, fanartImages), }; } } diff --git a/src/util/video-mime.js b/src/util/video-mime.js index 32c0d9f..43b3edb 100644 --- a/src/util/video-mime.js +++ b/src/util/video-mime.js @@ -18,11 +18,13 @@ class VideoMimeParser { return { container: "application/dash+xml", tracks: info.MPD.Period.AdaptationSet.map((track, i) => { + const representations = track.Representation instanceof Array ? track.Representation : [track.Representation]; + return { id: i, type: track.mimeType.substr(0, track.mimeType.indexOf('/')), - codec: [...new Set(track.Representation.map(r => r.codecs))].join(","), - bitrate: track.Representation.map(r => r.bandwidth).sort()[0], + codec: [...new Set(representations.map(r => r.codecs))].join(","), + bitrate: representations.map(r => r.bandwidth).sort()[0], language: track.lang } }) -- GitLab