Skip to content
Snippets Groups Projects
Verified Commit 41d3bc7e authored by Janne Mareike Koschinski's avatar Janne Mareike Koschinski
Browse files

Initial Commit

parents
No related branches found
No related tags found
No related merge requests found
/node_modules/
/.idea/
This diff is collapsed.
{
"name": "metadata_downloader",
"version": "1.0.0",
"description": "",
"main": "src/index.js",
"scripts": {
"start": "node --experimental-modules --es-module-specifier-resolution=node src/index.js"
},
"author": "",
"license": "MPL",
"type": "module",
"dependencies": {
"fs": "0.0.1-security",
"mp4box": "^0.3.20",
"node-fetch": "^2.6.0",
"node-tvdb": "^4.1.0",
"path": "^0.12.7",
"pnormaldist": "^1.0.1",
"sqlite3": "^4.1.1",
"typescript": "^3.8.3",
"uuid": "^7.0.3"
}
}
import fetch from 'node-fetch';
class FanartApi {
apiKey;
baseUrl;
constructor(apiKey, baseUrl) {
this.apiKey = apiKey;
this.baseUrl = baseUrl || "http://webservice.fanart.tv/v3/";
}
request(path, options) {
const url = new URL(path, this.baseUrl);
const params = new URLSearchParams(options || {});
params.append("api_key", this.apiKey);
url.search = params.toString();
return fetch(
url.href
).then((response) => {
return response.text().then(text => {
return {
ok: response.ok,
body: text,
}
});
}).then((data) => {
const {ok, body} = data;
if (!ok) {
throw new Error(`${url}: ${body}`);
}
return JSON.parse(body);
});
}
}
export default FanartApi;
\ No newline at end of file
import sqlite3 from 'sqlite3';
class ImdbApi {
database;
constructor(path) {
this.database = new sqlite3.Database(path);
}
query(query, args) {
return promisify(this.database, "get", query, args);
}
queryJson(query, args) {
return this.query(query, args)
.then((data) => data.json)
.then((data) => JSON.parse(data));
}
findTypeById(id) {
return this.query(ImdbApi.queryType, {
1: id
}).then(data => data.titleType);
}
findById(id) {
return this.queryJson(ImdbApi.queryGet, {
1: id
});
}
search(type, title, year) {
return this.queryJson(ImdbApi.querySearch, {
1: type,
2: title, 3: title,
4: year
});
}
static querySearch = `
SELECT tconst
FROM title
WHERE title.titleType = ?
AND (title.primaryTitle = ? OR title.originalTitle = ?)
AND title.startYear = ?
LIMIT 1
`
static queryType = `
SELECT title.titleType
FROM title
WHERE title.tconst = ?
`
static queryGet = `
SELECT json_object(
'id', title.tconst,
'titleType', title.titleType,
'primaryTitle', title.primaryTitle,
'originalTitle', title.originalTitle,
'isAdult', json(case when title.isAdult = 0 then 'false' else 'true' end),
'startYear', title.startYear,
'endYear', title.endYear,
'runtimeMinutes', title.runtimeMinutes,
'genres', json('["' || replace(title.genres, ',', '","') || '"]'),
'rating', json_object(
'averageRating', title_ratings.averageRating,
'numVotes', title_ratings.numVotes
),
'crew', json_object(
'directors', json('["' || replace(title_crew.directors, ',', '","') || '"]'),
'writers', json('["' || replace(title_crew.writers, ',', '","') || '"]')
),
'aka', json(aka.aka),
'principals', json(principals.principals),
'episodes', json(episode.episode)
) AS json
FROM title
LEFT OUTER JOIN (SELECT title_principals.tconst,
json_group_array(json_object(
'person', json_object(
'nconst', name.nconst,
'primaryName', name.primaryName,
'birthYear', name.birthYear,
'deathYear', name.deathYear,
'primaryProfession',
json('["' || replace(name.primaryProfession, ',', '","') || '"]'),
'knownForTitles',
json('["' || replace(name.knownForTitles, ',', '","') || '"]')
),
'category', title_principals.category,
'job', title_principals.job,
'characters', json(title_principals.characters)
)) AS principals
FROM title_principals
LEFT OUTER JOIN name on title_principals.nconst = name.nconst
GROUP BY title_principals.tconst) AS principals ON title.tconst = principals.tconst
LEFT OUTER JOIN (SELECT title_aka.titleId,
json_group_array(json_object(
'title', title_aka.title,
'region', title_aka.region,
'languages', title_aka.language,
'types', json('["' || replace(title_aka.types, ',', '","') || '"]'),
'attributes', json('["' || replace(title_aka.attributes, ',', '","') || '"]'),
'isOriginalTitle',
json(case when title_aka.isOriginalTitle = 0 then 'false' else 'true' end)
)) AS aka
FROM title_aka
GROUP BY title_aka.titleId) AS aka ON title.tconst = aka.titleId
LEFT OUTER JOIN (SELECT title_episode.parentTconst,
json_group_array(json_object(
'tconst', title_episode.tconst,
'season', title_episode.seasonNumber,
'episode', title_episode.episodeNumber
)) AS episode
FROM title_episode
GROUP BY title_episode.parentTconst) AS episode ON title.tconst = episode.parentTconst
LEFT OUTER JOIN title_ratings on title.tconst = title_ratings.tconst
LEFT OUTER JOIN title_crew on title.tconst = title_crew.tconst
WHERE title.tconst = ?
`;
}
function promisify(db, fun) {
const args = Array.prototype.slice.call(arguments).slice(2);
return new Promise((resolve, reject) => {
db[fun](...args, function (err, row) {
if (err) {
reject(err);
} else {
resolve(row);
}
})
});
}
export default ImdbApi;
\ No newline at end of file
import fetch from 'node-fetch';
class TmdbApi {
apiKey;
baseUrl;
constructor(apiKey, baseUrl) {
this.apiKey = apiKey;
this.baseUrl = baseUrl || "https://api.themoviedb.org/3/";
}
async updateConfiguration() {
this.configuration = await this.request('configuration');
return this.configuration;
}
getImageUrl(path) {
return new URL(`original${path}`, this.configuration.images.secure_base_url).href;
}
request(path, options) {
const url = new URL(path, this.baseUrl);
url.search = new URLSearchParams(options || {}).toString();
return fetch(url.href, {
headers: {
'Authorization': `Bearer ${this.apiKey}`,
'Content-Type': 'application/json;charset=utf-8',
}
}).then((response) => {
return response.text().then(text => {
return {
ok: response.ok,
body: text,
}
});
}).then((data) => {
const {ok, body} = data;
if (!ok) {
throw new Error(`${url}: ${body}`);
}
return JSON.parse(body);
});
}
}
export default TmdbApi;
\ No newline at end of file
import path from 'path';
import {promises as fsPromises} from 'fs';
import parseMediaInfo from "./util/video-mime";
class FileManager {
basePath;
moviesPath;
showsPath;
constructor(basePath) {
this.basePath = basePath;
}
async updateConfiguration() {
this.moviesPath = await this.findPath(this.basePath, "movies");
this.showsPath = await this.findPath(this.basePath, "shows");
}
async findPath(base, name) {
const children = await fsPromises.readdir(base);
const result = children.find(child => {
return child.localeCompare(name, "en", {
sensitivity: "base", usage: "search", ignorePunctuation: true
}) === 0
})
return result ? path.join(base, result) : null;
}
async listMovies() {
return fsPromises
.readdir(this.moviesPath, {withFileTypes: true})
.then(result => result
.filter(dir => dir.isDirectory())
.map(dir => dir.name)
);
}
async listShows() {
return fsPromises
.readdir(this.showsPath, {withFileTypes: true})
.then(result => result
.filter(dir => dir.isDirectory())
.map(dir => dir.name)
);
}
async findMedia(base) {
const files = await fsPromises
.readdir(base, {withFileTypes: true})
.then(result => result
.filter(dir => dir.isFile())
.map(dir => dir.name)
);
const dashManifest = files.find(fileName => fileName.endsWith(".mpd"))
const subtitleBase = path.join(base, "subtitles");
const subtitleFiles = (await fsPromises
.readdir(subtitleBase, {withFileTypes: true})
.then(result => result
.filter(dir => dir.isFile())
.map(dir => dir.name)
).catch(err => {
// Do nothing, just means file does not exist
})) || [];
const subtitles = subtitleFiles.filter(fileName =>
fileName.endsWith(".srt") ||
fileName.endsWith(".ttml") ||
fileName.endsWith(".ass") ||
fileName.endsWith(".vtt")
)
const mediaFiles = dashManifest ? [] : files.filter(fileName =>
fileName.endsWith(".mp4") ||
fileName.endsWith(".webm") ||
fileName.endsWith(".ogg")
)
return {
subtitles: subtitles.map(name => {
const {language, region, specifier, format} = /^(?<language>\p{L}+)(?:-(?<region>\p{L}+))?(?:\.(?<specifier>.+))?\.(?<format>[^.]+)$/u.exec(name).groups;
return {
language: language,
region: region,
specifier: specifier,
format: format,
src: path.join("subtitles", name)
}
}),
media: dashManifest ? [{
src: dashManifest,
container: "application/xml+dash"
}] : await Promise.all(mediaFiles.map(fileName => parseMediaInfo(path.join(base, fileName)).then(metadata => {
return {
src: fileName,
...metadata
}
})))
}
}
}
export default FileManager;
\ No newline at end of file
import process from 'process';
import path from 'path';
import ImdbApi from './api/imdb_api';
import TmdbApi from './api/tmdb_api';
import TvdbApi from 'node-tvdb';
import FanartApi from './api/fanart_api';
import MetadataLoader from "./metadata_loader";
import FileManager from "./directory_walker";
import parseMediaInfo from "./util/video-mime";
async function main() {
const args = process.argv.slice(2);
const basePath = args[0];
const imdbApi = new ImdbApi(process.env.IMDB_PATH);
const tmdbApi = new TmdbApi(process.env.TMDB_API_KEY);
await tmdbApi.updateConfiguration();
const tvdbApi = new TvdbApi(process.env.TVDB_API_KEY);
const fanartApi = new FanartApi(process.env.FANART_API_KEY);
const loader = new MetadataLoader(imdbApi, tmdbApi, tvdbApi, fanartApi);
const fileManager = new FileManager(basePath);
await fileManager.updateConfiguration();
console.log(JSON.stringify(
await fileManager.findMedia(path.join(fileManager.moviesPath, "Avengers: Endgame (2019)")),
null, 2
));
console.log(JSON.stringify(
await fileManager.findMedia(path.join(fileManager.showsPath, "Steins;Gate (2011)", "S01E01")),
null, 2
));
console.log(JSON.stringify(
await parseMediaInfo(path.join(fileManager.showsPath, "Steins;Gate (2011)", "S01E01", "1080-fragment.mp4")),
null, 2
));
/*
let directoryPath = args[0];
if (directoryPath) {
const showsPath = path.join(directoryPath, "Shows");
const moviesPath = path.join(directoryPath, "Movies");
if (fs.existsSync(showsPath)) {
const shows = await fs.promises.readdir(showsPath);
for (let fileName of shows) {
const filePath = path.join(showsPath, fileName);
try {
console.log("Processing show ", filePath);
loader.processFile(true, tmdbApi, fanartApi, imdb, tvdbApi, uuid.v4(), filePath);
} catch (e) {
console.error(e);
}
}
}
if (fs.existsSync(moviesPath)) {
const movies = await fs.promises.readdir(moviesPath);
for (let fileName of movies) {
const filePath = path.join(moviesPath, fileName);
try {
console.log("Processing movie ", filePath);
loader.processFile(false, tmdbApi, fanartApi, imdb, tvdbApi, uuid.v4(), filePath);
} catch (e) {
console.error(e);
}
}
}
}
*/
}
(async function () {
await main()
}());
\ No newline at end of file
import ranking_confidence from './util/statistics';
class MetadataLoader {
imdb;
tmdb;
tvdb;
fanart;
constructor(imdb, tmdb, tvdb, fanart) {
this.imdb = imdb;
this.tmdb = tmdb;
this.tvdb = tvdb;
this.fanart = fanart;
}
transformData(ids, imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations) {
return {
ids: ids,
originalLanguage: tmdbResult.original_language,
originalTitle: imdbResult.originalTitle,
primaryTitle: imdbResult.primaryTitle,
titles: imdbResult.aka
.filter(el => el.types !== null && el.types.includes("imdbDisplay") === true)
.map(el => {
return {
title: el.title,
region: el.region,
languages: el.languages ? el.languages.split(",") : [],
}
}),
primaryDescription: {
overview: tmdbResult.overview,
tagline: tmdbResult.tagline,
},
descriptions: tmdbTranslations.translations.map(el => {
return {
region: el.iso_3166_1,
languages: el.iso_639_1 ? el.iso_639_1.split(",") : [],
overview: el.data.overview,
tagline: el.data.tagline,
}
}).filter(el => el.overview),
yearStart: imdbResult.startYear,
yearEnd: imdbResult.endYear,
runtime: imdbResult.runtimeMinutes,
seasons: imdbResult.seasons,
episodes: imdbResult.episodes,
genres: tmdbResult.genres.map(el => el.name),
cast: imdbResult.principals.map(el => {
return {
id: el.person.nconst,
name: el.person.primaryName,
category: el.category,
job: el.job,
characters: el.characters,
}
}),
ratings: tmdbContentRatings.results.map(el => {
const certification =
Array.isArray(el) ? el.release_dates.sort((a, b) => b.type - a.type).map(el => el.certification) :
el ? el.rating :
null;
return {
region: el.iso_3166_1,
certification: certification,
}
}).filter(el => el.certification)
}
}
chooseImages(originalLanguage, tmdbImages, fanartImages) {
function findbest(list) {
let sorted = list.sort((a, b) => b.likes - a.likes);
return sorted.find(el => el.language === originalLanguage) || sorted[0];
}
function calculateConfidenceAndQuality(containsText, originalLanguage) {
return (element) => {
return {
confidence: ranking_confidence(element.vote_average, element.vote_count),
lang_quality: containsText ? (
element.iso_639_1 === originalLanguage ? 2 :
element.iso_639_1 === null ? 1 :
0
) : (
element.iso_639_1 === null ? 2 :
element.iso_639_1 === originalLanguage ? 1 :
0
),
...element
}
}
}
function imageComparator(a, b) {
return (b.lang_quality * 10 + b.confidence) - (a.lang_quality * 10 + a.confidence)
}
return {
logo: fanartImages && fanartImages.hdmovielogo ? findbest(fanartImages.hdmovielogo) : null,
poster: tmdbImages.posters.map(calculateConfidenceAndQuality(true, originalLanguage))
.sort(imageComparator)[0],
backdrop: tmdbImages.backdrops.map(calculateConfidenceAndQuality(false, originalLanguage))
.sort(imageComparator)[0],
}
}
async loadMetadata(ids) {
const titleType = await this.imdb.findTypeById(ids.imdb);
const tmdbSources = titleType === "tvSeries" ?
[`tv/${ids.tmdb}`, `tv/${ids.tmdb}/content_ratings`, `tv/${ids.tmdb}/translations`, `tv/${ids.tmdb}/images`] :
[`movie/${ids.tmdb}`, `movie/${ids.tmdb}/release_dates`, `movie/${ids.tmdb}/translations`, `movie/${ids.tmdb}/images`]
const fanartSource = titleType === "tvSeries" ?
`tv/${ids.tmdb}` :
`movies/${ids.tmdb}`;
const [imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations, tmdbImages, fanartImages] = await Promise.all([
this.imdb.findById(ids.imdb),
...tmdbSources.map(url => this.tmdb.request(url)),
this.fanart.request(fanartSource).catch((e) => {
// do nothing, it just means it wasn’t found
})
]);
const metadata = this.transformData(ids, imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations);
return {
metadata: metadata,
// also use tvdb for images
images: this.chooseImages(metadata.originalLanguage, tmdbImages, fanartImages),
};
}
}
export default MetadataLoader;
\ No newline at end of file
// FIXME: Implement all this over to the metadata loader, and clean it up meanwhile
async function identifyMovie(api, imdb, tvdb, title, year) {
const results = await api.request(`search/movie`, {
query: title,
primary_release_year: year
}).catch((e) => console.error(e));
const result = results.results.sort((a, b) => {
return b.popularity - a.popularity;
})[0];
return result ? {
tmdb: result.id
} : null;
}
async function identifyShow(tmdbApi, imdbApi, tvdbApi, title, showYear) {
const imdbId = await utils.promisify(imdbApi, "get", idQuery, {1: "tvSeries", 2: title, 3: title, 4: showYear})
.then((data) => data.tconst);
const tvdbResults = await tvdbApi.getSeriesByImdbId(imdbId).catch((e) => console.error(e));
const tvdbResult = tvdbResults[0];
if (!tvdbResult) return null;
const tmdbResults = await tmdbApi.request(`find/${imdbId}`, {
"external_source": "imdb_id",
}).catch((e) => console.error(e));
const tmdbResult = tmdbResults.tv_results.sort((a, b) => {
return b.popularity - a.popularity;
})[0];
if (!tmdbResult) return null;
return {
imdb: imdbId,
tvdb: tvdbResult.id,
tmdb: tmdbResult.id,
};
}
function downloadImage(url, filePath) {
return new Promise(((resolve, reject) => {
const file = fs.createWriteStream(filePath);
https.get(url, function (response) {
response.pipe(file);
file.on('finish', function () {
file.close(() => {
resolve();
});
});
}).on('error', function (err) {
fs.unlink(filePath);
reject(err);
});
}));
}
async function processSubtitles(filePath) {
const subtitlesPath = path.join(filePath, 'subtitles');
let subtitles;
if (fs.existsSync(subtitlesPath)) {
const files = await fs.promises.readdir(subtitlesPath);
subtitles = files.map((filename) => {
const match = /^(?<language>.+)(?<specifier>-.+)?\.(?<type>.*)$/.exec(path.basename(filename));
if (match) {
const {language, specifier, type} = match.groups;
return {
language: language,
specifier: specifier,
type: type,
}
} else {
return null;
}
}).filter(el => el);
}
return subtitles;
}
async function processFile(isShow, tmdbApi, fanartApi, imdb, tvdbApi, uuid, filePath) {
const {name, year} = /^(?<name>.+) \((?<year>\d+)\)$/.exec(path.basename(filePath)).groups;
const identifyFunction = isShow ? identifyShow : identifyMovie;
const ids = await identifyFunction(tmdbApi, imdb, tvdbApi, name, year);
if (ids === null) {
console.error(`No item in TMDB found for "${name}" from year "${year}" at path "${filePath}"`);
return
}
function processImage(type, imageUrl) {
const ending = path.extname(imageUrl);
return downloadImage(imageUrl, path.join(filePath, `metadata/${type}${ending}`));
}
data.data.subtitles = processSubtitles(filePath);
data.data.hasLogo = !!hdMovieLogo;
data.data.src = path.join("/api/", isShow ? "Shows" : "Movies", encodeURIComponent(path.basename(filePath)));
// Write metadata
await utils.promisify(fs, "writeFile", path.join(filePath, "metadata.json"), JSON.stringify(data.data));
// Creating metadata folder
if (!fs.existsSync(path.join(filePath, 'metadata'))) {
await fs.promises.mkdir(path.join(filePath, 'metadata'));
}
// Writing images
await Promise.all([
poster ? processImage("poster", tmdbApi.getImageUrl(poster.file_path)) : null,
backdrop ? processImage("backdrop", tmdbApi.getImageUrl(backdrop.file_path)) : null,
hdMovieLogo ? processImage("logo", hdMovieLogo.url) : null,
].filter(el => el));
}
\ No newline at end of file
function promisify(fun) {
const args = Array.prototype.slice.call(arguments).slice(1);
return new Promise((resolve, reject) => {
fun(...args, function (err) {
const args = Array.prototype.slice.call(arguments).slice(1);
if (err) {
reject(err);
} else {
resolve(args);
}
})
});
}
export default promisify;
\ No newline at end of file
import pnormaldist from 'pnormaldist';
const DEFAULT_CONFIDENCE = 0.95;
function ranking_confidence(value, n, confidence) {
if (n === 0) return 0;
if (confidence === undefined) confidence = DEFAULT_CONFIDENCE;
const z = pnormaldist(1 - (1 - confidence) / 2);
const phat = 1.0 * value / n;
const result = (phat + z * z / (2 * n) - z * Math.sqrt((phat * (1 - phat) + z * z / (4 * n)) / n)) / (1 + z * z / n);
if (isNaN(result) || !isFinite(result)) {
return 0;
} else {
return result;
}
}
export default ranking_confidence;
\ No newline at end of file
import fs from 'fs';
import path from 'path';
import {exec} from "child_process";
import MP4Box from 'mp4box';
import promisify from "./promisify";
async function parseMediaInfoWebm(filePath) {
const [stdout] = await promisify(exec, `ffprobe -print_format json -show_format -show_streams -bitexact "${filePath}"`);
return JSON.parse(stdout);
}
function processMediaInfoWebm(info) {
console.log(JSON.stringify(info, null, 2));
return {
container: "video/webm",
tracks: {
video: info.streams.filter(track => track.codec_type === "video").map(track => {
return {
// FIXME: this is broken thanks to ffmpeg
codec: `${track.codec_name}.${track.profile}.${track.level}.`
}
}),
audio: info.streams.filter(track => track.codec_type === "audio").map(track => {
return {
}
})
}
};
}
function parseMediaInfoMp4(filePath) {
return new Promise((resolve, reject) => {
const mp4boxfile = MP4Box.createFile(); /* eslint-disable-line new-cap */
const stream = fs.createReadStream(filePath);
mp4boxfile.onReady = (info) => {
resolve(info);
stream.close();
};
mp4boxfile.onError = err => stream.destroy(err);
let offset = 0;
stream.on('data', function (chunk) {
var arrayBuffer = new Uint8Array(chunk).buffer;
arrayBuffer.fileStart = offset;
offset += chunk.byteLength;
mp4boxfile.appendBuffer(arrayBuffer);
});
stream.on('error', reject);
stream.on('end', () => reject(new Error('Invalid file type.')));
});
}
function processMediaInfoMp4(info) {
return {
container: "video/mp4",
duration: info.duration,
tracks: {
video: info.videoTracks.map(track => {
return {
id: track.id,
bitrate: Math.round(track.bitrate),
language: track.language,
codec: track.codec,
}
}),
audio: info.audioTracks.map(track => {
return {
id: track.id,
bitrate: Math.round(track.bitrate),
language: track.language,
codec: track.codec,
}
})
}
}
}
async function parseMediaInfo(filePath) {
switch (path.extname(filePath)) {
case '.mp4':
return processMediaInfoMp4(await parseMediaInfoMp4(filePath));
case '.webm':
return processMediaInfoWebm(await parseMediaInfoWebm(filePath));
default:
console.error(`Invalid extension: ${path.extname(filePath)} for path ${filePath}`)
return null;
}
}
export default parseMediaInfo
\ No newline at end of file
interface TvdbSeriesResponse {
"data": {
"added": string,
"airsDayOfWeek": string,
"airsTime": string,
"aliases": Array<String>,
"banner": string,
"firstAired": string,
"genre": Array<String>,
"id": number,
"imdbId": string,
"lastUpdated": number,
"network": string,
"networkId": string,
"overview": string,
"rating": string,
"runtime": string,
"seriesId": string,
"seriesName": string,
"siteRating": number,
"siteRatingCount": number,
"slug": string,
"status": string,
"zap2itId": string
} | null,
"errors": {
"invalidFilters": [
string
],
"invalidLanguage": string,
"invalidQueryParams": [
string
]
} | null
}
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment