From 2b852ce83b74634e9727c8ee507fab3de2cc2448 Mon Sep 17 00:00:00 2001 From: Janne Koschinski <janne@kuschku.de> Date: Sat, 9 May 2020 00:30:10 +0200 Subject: [PATCH] Implement database backend --- package-lock.json | 256 ++++++++++++++++++++++++++ package.json | 3 + src/index.js | 107 ++--------- src/metadata_loader.js | 372 +++++++++++++++++++++++++++++--------- src/model.js | 37 ++++ src/process_content.js | 61 +++++++ src/storage.js | 287 +++++++++++++++++++++++++++++ src/util/download-file.js | 2 +- src/util/video-mime.js | 9 +- 9 files changed, 952 insertions(+), 182 deletions(-) create mode 100644 src/model.js create mode 100644 src/process_content.js create mode 100644 src/storage.js diff --git a/package-lock.json b/package-lock.json index 3452f79..3ca2f92 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,11 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/node": { + "version": "13.13.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.5.tgz", + "integrity": "sha512-3ySmiBYJPqgjiHA7oEaIo2Rzz0HrOZ7yrNO5HWyaE5q0lQ3BppDZ3N53Miz8bw2I7gh1/zir2MGVZBvpb1zq9g==" + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -25,6 +30,11 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", @@ -88,6 +98,11 @@ "file-uri-to-path": "1.0.0" } }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -97,6 +112,11 @@ "concat-map": "0.0.1" } }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", @@ -107,6 +127,15 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, + "cls-bluebird": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cls-bluebird/-/cls-bluebird-2.1.0.tgz", + "integrity": "sha1-N+8eCAqP+1XC9BZPU28ZGeeWiu4=", + "requires": { + "is-bluebird": "^1.0.2", + "shimmer": "^1.1.0" + } + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -171,6 +200,11 @@ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" }, + "dottie": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz", + "integrity": "sha512-fmrwR04lsniq/uSr8yikThDTrM7epXHBAAjH9TbeH3rEA8tdCO7mRzB9hdmdGyJCxF8KERo9CITcm3kGuoyMhg==" + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -324,6 +358,11 @@ "minimatch": "^3.0.4" } }, + "inflection": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz", + "integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY=" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -343,6 +382,11 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, + "is-bluebird": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-bluebird/-/is-bluebird-1.0.2.tgz", + "integrity": "sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI=" + }, "is-fullwidth-code-point": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", @@ -478,6 +522,19 @@ "minimist": "^1.2.5" } }, + "moment": { + "version": "2.25.3", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.25.3.tgz", + "integrity": "sha512-PuYv0PHxZvzc15Sp8ybUCoQ+xpyPWvjOuK72a5ovzp2LI32rJXOiIfyoFoYvG3s6EwwrdkMyWuRiEHSZRLJNdg==" + }, + "moment-timezone": { + "version": "0.5.28", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz", + "integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==", + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -623,6 +680,11 @@ "os-tmpdir": "^1.0.0" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "path": { "version": "0.12.7", "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", @@ -642,11 +704,104 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, + "pg": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz", + "integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-packet-stream": "^1.1.0", + "pg-pool": "^2.0.10", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + }, + "dependencies": { + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + } + } + }, + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + }, + "pg-hstore": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.3.tgz", + "integrity": "sha512-qpeTpdkguFgfdoidtfeTho1Q1zPVPbtMHgs8eQ+Aan05iLmIs3Z3oo5DOZRclPGoQ4i68I1kCtQSJSa7i0ZVYg==", + "requires": { + "underscore": "^1.7.0" + } + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-packet-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pg-packet-stream/-/pg-packet-stream-1.1.0.tgz", + "integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg==" + }, + "pg-pool": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz", + "integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", + "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "requires": { + "split": "^1.0.0" + } + }, "pnormaldist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pnormaldist/-/pnormaldist-1.0.1.tgz", "integrity": "sha1-pda4vyF1xQhOvvJDDHn3N5+EMM4=" }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.5.tgz", + "integrity": "sha512-pdau6GRPERdAYUQwkBnGKxEfPyhVZXG/JiS44iZWiNdSOWE09N2lUgN6yshuq6fVSon4Pm0VMXd1srUUkLe9iA==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -731,6 +886,14 @@ } } }, + "retry-as-promised": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz", + "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==", + "requires": { + "any-promise": "^1.3.0" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -759,16 +922,76 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" }, + "sequelize": { + "version": "5.21.7", + "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.21.7.tgz", + "integrity": "sha512-+JrS5Co7CN53cOFFFaUb+xqQP00wD1Ag9xGLBLoUko2KhRZxjm+UDkqAVPHTUp87McLwJaCPkKv61GPhBVloRQ==", + "requires": { + "bluebird": "^3.5.0", + "cls-bluebird": "^2.1.0", + "debug": "^4.1.1", + "dottie": "^2.0.0", + "inflection": "1.12.0", + "lodash": "^4.17.15", + "moment": "^2.24.0", + "moment-timezone": "^0.5.21", + "retry-as-promised": "^3.2.0", + "semver": "^6.3.0", + "sequelize-pool": "^2.3.0", + "toposort-class": "^1.0.1", + "uuid": "^3.3.3", + "validator": "^10.11.0", + "wkx": "^0.4.8" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, + "sequelize-pool": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-2.3.0.tgz", + "integrity": "sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA==" + }, "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, + "shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + }, "signal-exit": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, "sqlite3": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.1.1.tgz", @@ -840,6 +1063,11 @@ "yallist": "^3.0.3" } }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, "topo": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/topo/-/topo-3.0.3.tgz", @@ -855,6 +1083,11 @@ } } }, + "toposort-class": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", + "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg=" + }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -882,6 +1115,11 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==" }, + "underscore": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.10.2.tgz", + "integrity": "sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg==" + }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", @@ -908,6 +1146,11 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz", "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==" }, + "validator": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz", + "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw==" + }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", @@ -926,6 +1169,14 @@ "string-width": "^1.0.2 || 2" } }, + "wkx": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz", + "integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==", + "requires": { + "@types/node": "*" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -941,6 +1192,11 @@ "node-expat": "^2.3.18" } }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/package.json b/package.json index 7979757..a15dd80 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,10 @@ "node-fetch": "^2.6.0", "node-tvdb": "^4.1.0", "path": "^0.12.7", + "pg": "^7.18.2", + "pg-hstore": "^2.3.3", "pnormaldist": "^1.0.1", + "sequelize": "^5.21.7", "sqlite3": "^4.1.1", "typescript": "^3.8.3", "uuid": "^7.0.3", diff --git a/src/index.js b/src/index.js index af8e389..e42d80a 100644 --- a/src/index.js +++ b/src/index.js @@ -1,7 +1,5 @@ import process from 'process'; - -import path from 'path'; -import {promises as fsPromises} from 'fs'; +import sequelize from "sequelize"; import ImdbApi from './api/imdb_api'; import TmdbApi from './api/tmdb_api'; @@ -12,6 +10,9 @@ import MetadataLoader from "./metadata_loader"; import FileManager from "./directory_walker"; import VideoMimeParser from "./util/video-mime"; +import Backend from "./storage"; +import processContent from "./process_content"; + async function main() { const args = process.argv.slice(2); const basePath = args[0]; @@ -22,95 +23,25 @@ async function main() { const tvdbApi = new TvdbApi(process.env.TVDB_API_KEY); const fanartApi = new FanartApi(process.env.FANART_API_KEY); + const storage = new Backend(new sequelize.Sequelize( + process.env.DB_DATABASE, + process.env.DB_USERNAME, + process.env.DB_PASSWORD, + { + dialect: "postgres", + host: process.env.DB_HOST, + port: +process.env.DB_PORT, + ssl: !!process.env.DB_SSL, + } + )); + await storage.sync(); + const videoMimeParser = new VideoMimeParser(process.env.MP4INFO_PATH || "mp4info", process.env.FFPROBE_PATH || "ffprobe"); - const loader = new MetadataLoader(imdbApi, tmdbApi, tvdbApi, fanartApi); + const loader = new MetadataLoader(imdbApi, tmdbApi, tvdbApi, fanartApi, storage); const fileManager = new FileManager(basePath, videoMimeParser); await fileManager.updateConfiguration(); - async function processMovie(filePath) { - const {name, year} = /^(?<name>.+) \((?<year>\d+)\)$/.exec(path.basename(filePath)).groups; - - const ids = (await fileManager.findIds(filePath)) || (await loader.identifyMovie(name, year)); - if (!ids) { - console.error(`Could not identify movie ${name} (${year}) at ${filePath}`) - return; - } - const [media, {metadata, images}] = await Promise.all([ - fileManager.findMedia(filePath), - loader.loadMetadata(ids), - ]); - const imageData = await loader.processImages(basePath, filePath, images); - - await Promise.all([ - fsPromises.writeFile(path.join(filePath, "ids.json"), JSON.stringify(ids, null, 2)), - fsPromises.writeFile(path.join(filePath, "metadata.json"), JSON.stringify({ - ...metadata, - ...media, - images: imageData.map(img => { - return { - type: img.type, - src: img.src, - } - }), - }, null, 2)), - ]); - } - - async function processEpisode(showIds, episodeIdentifier, filePath) { - const [media, {metadata, images}] = await Promise.all([ - fileManager.findMedia(filePath), - loader.loadEpisodeMetadata(showIds, episodeIdentifier), - ]); - const imageData = await loader.processImages(basePath, filePath, images); - - await Promise.all([ - fsPromises.writeFile(path.join(filePath, "metadata.json"), JSON.stringify({ - ...metadata, - ...media, - images: imageData.map(img => { - return { - type: img.type, - src: img.src, - } - }), - }, null, 2)), - ]); - } - - async function processShow(filePath) { - const {name, year} = /^(?<name>.+) \((?<year>\d+)\)$/.exec(path.basename(filePath)).groups; - const ids = (await fileManager.findIds(filePath)) || (await loader.identifyShow(name, year)); - if (!ids) { - console.error(`Could not identify show ${name} (${year}) at ${filePath}`) - return; - } - const episodes = await fileManager.listEpisodes(filePath); - - const {metadata, images} = await loader.loadMetadata(ids); - const imageData = await loader.processImages(basePath, filePath, images); - await Promise.all([ - ...episodes.map(async ({episodeIdentifier, filePath}) => await processEpisode(ids, episodeIdentifier, filePath).catch(err => { - console.error(`Error processing episode ${JSON.stringify(episodeIdentifier)} of show ${JSON.stringify(ids)}: `, err); - })), - fsPromises.writeFile(path.join(filePath, "ids.json"), JSON.stringify(ids, null, 2)), - fsPromises.writeFile(path.join(filePath, "metadata.json"), JSON.stringify({ - ...metadata, - episodes: episodes.map(episode => { - return { - src: episode.src, - episodeIdentifier: episode.episodeIdentifier, - } - }), - images: imageData, - }, null, 2)), - ]); - } - - const [movies, shows] = await Promise.all([fileManager.listMovies(), fileManager.listShows()]); - await Promise.all([ - ...movies.map(processMovie), - ...shows.map(processShow) - ]); + await processContent(basePath, fileManager, loader); } (async function () { diff --git a/src/metadata_loader.js b/src/metadata_loader.js index 9745d9e..9e08464 100644 --- a/src/metadata_loader.js +++ b/src/metadata_loader.js @@ -3,104 +3,243 @@ import uuid from 'uuid'; import path from "path"; import downloadFile from "./util/download-file"; import encodePath from "./util/encode-path"; +import { + Genre, + Person, + Title, + TitleCast, + TitleDescription, + TitleEpisode, + TitleGenre, + TitleImage, TitleMedia, + TitleName, + TitleRating, + TitleSubtitles +} from "./model"; class MetadataLoader { imdb; tmdb; tvdb; fanart; + storage; - constructor(imdb, tmdb, tvdb, fanart) { + constructor(imdb, tmdb, tvdb, fanart, storage) { this.imdb = imdb; this.tmdb = tmdb; this.tvdb = tvdb; this.fanart = fanart; + this.storage = storage; } - transformData(ids, imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations) { - return { - ids: ids, - originalLanguage: tmdbResult.original_language, - originalTitle: imdbResult.originalTitle, - primaryTitle: imdbResult.primaryTitle, - titles: imdbResult.aka - .filter(el => el.types !== null && el.types.includes("imdbDisplay") === true) - .map(el => { - return { - title: el.title, - region: el.region, - languages: el.languages ? el.languages.split(",") : [], - } - }), - primaryDescription: { - overview: tmdbResult.overview, - tagline: tmdbResult.tagline, - }, - descriptions: tmdbTranslations.translations.map(el => { - return { - region: el.iso_3166_1, - languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], - overview: el.data.overview, - tagline: el.data.tagline, - } - }).filter(el => el.overview), - yearStart: imdbResult.startYear, - yearEnd: imdbResult.endYear, - runtime: imdbResult.runtimeMinutes, - seasons: imdbResult.seasons, - episodes: imdbResult.episodes, - genres: tmdbResult.genres.map(el => el.name), - cast: imdbResult.principals.map(el => { - return { - id: el.person.nconst, - name: el.person.primaryName, - category: el.category, - job: el.job, - characters: el.characters, - } - }), - ratings: tmdbContentRatings.results.map(el => { - const certification = - Array.isArray(el.release_dates) ? el.release_dates.sort((a, b) => b.type - a.type).map(el => el.certification)[0] : + async transformData(ids, imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations) { + const [title] = await Title.upsert({ + id: ids.uuid, + imdb_id: ids.imdb, + tmdb_id: ids.tmdb, + tvdb_id: ids.tvdb, + original_language: tmdbResult.original_language, + runtime: imdbResult.runtime, + year_start: imdbResult.startYear, + year_end: imdbResult.endYear, + }, {returning: true}); + await TitleName.destroy({ + where: { + title_id: title.id, + } + }) + const primaryTitleName = await TitleName.build({ + region: null, + languages: null, + original: false, + name: imdbResult.primaryTitle, + }); + await primaryTitleName.setTitle(title.id, {save: false}); + await primaryTitleName.save(); + const originalTitleName = await TitleName.build({ + region: null, + languages: null, + original: true, + name: imdbResult.originalTitle, + }); + await originalTitleName.setTitle(title.id, {save: false}); + await originalTitleName.save(); + for (let el of imdbResult.aka.filter(el => el.types !== null && el.types.includes("imdbDisplay") === true)) { + const titleName = await TitleName.build({ + region: el.region, + languages: el.languages ? el.languages.split(",") : [], + original: false, + name: el.title, + }) + await titleName.setTitle(title.id, {save: false}); + await titleName.save(); + } + await TitleDescription.destroy({ + where: { + title_id: title.id, + } + }) + const originalTitleDescription = await TitleDescription.build({ + region: null, + languages: null, + overview: tmdbResult.overview, + tagline: tmdbResult.tagline, + }); + await originalTitleDescription.setTitle(title.id, {save: false}); + await originalTitleDescription.save(); + for (let el of tmdbTranslations.translations) { + const titleDescription = await TitleDescription.build({ + region: el.iso_3166_1, + languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], + overview: el.data.overview, + tagline: el.data.tagline, + }) + await titleDescription.setTitle(title.id, {save: false}); + await titleDescription.save(); + } + await TitleCast.destroy({ + where: { + title_id: title.id, + } + }) + for (let el of imdbResult.principals) { + const [person] = await Person.upsert({ + imdb_id: el.person.nconst, + name: el.person.primaryName, + }, {returning: true}); + + const titleCast = await TitleCast.build({ + category: el.category, + characters: el.characters, + job: el.job, + }); + await titleCast.setTitle(title.id, {save: false}); + await titleCast.setPerson(person.id, {save: false}); + await titleCast.save(); + } + + await TitleGenre.destroy({ + where: { + title_id: title.id, + } + }) + for (let el of tmdbResult.genres) { + const [genre] = await Genre.upsert({ + tmdb_id: el.id, + name: el.name, + }, {returning: true}); + + const titleGenre = await TitleGenre.build({}); + await titleGenre.setTitle(title.id, {save: false}); + await titleGenre.setGenre(genre.id, {save: false}); + await titleGenre.save(); + } + + await TitleRating.destroy({ + where: { + title_id: title.id, + } + }) + for (let el of tmdbContentRatings.results) { + const certification = + Array.isArray(el.release_dates) ? el.release_dates.sort((a, b) => b.type - a.type).map(el => el.certification)[0] : el ? el.rating : - null; - return { - region: el.iso_3166_1, - certification: certification, - } - }).filter(el => el.certification) + null; + const titleRating = await TitleRating.build({ + region: el.iso_3166_1, + certification: certification, + title_id: title.id, + }); + await titleRating.setTitle(title.id, {save: false}); + await titleRating.save(); } + + return title; } - transformEpisodeData(ids, imdbResult, tmdbResult, tmdbTranslations) { + async transformEpisodeData(ids, episodeIdentifier, imdbResult, tmdbResult, tmdbTranslations) { if (!imdbResult) return null; if (!tmdbResult) return null; if (!tmdbTranslations) return null; - return { - ids: ids, - originalLanguage: tmdbResult.original_language, - originalTitle: imdbResult.originalTitle, - primaryTitle: imdbResult.primaryTitle, - titles: tmdbTranslations.translations.map(el => { - return { - title: el.data.name, - region: el.iso_3166_1, - languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], - } - }).filter(el => el.overview), - primaryDescription: { - overview: tmdbResult.overview, + const showTitle = await Title.findByPk(ids.uuid); + const [mapping] = await TitleEpisode.findOrBuild({ + where: { + show_id: showTitle.id, + season_number: episodeIdentifier.season, + episode_number: episodeIdentifier.episode, }, - descriptions: tmdbTranslations.translations.map(el => { - return { - region: el.iso_3166_1, - languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], - overview: el.data.overview, - } - }).filter(el => el.overview), - runtime: imdbResult.runtimeMinutes + defaults: { + episode_id: uuid.v4(), + } + }) + const [episodeTitle] = await Title.upsert({ + id: mapping.episode_id, + imdb_id: imdbResult.id, + tmdb_id: tmdbResult.id, + tvdb_id: null, + original_language: showTitle.original_language, + }, {returning: true}); + mapping.air_date = tmdbResult.air_date; + await mapping.setShow(showTitle.id, {save: false}); + await mapping.setEpisode(episodeTitle, {save: false}); + await mapping.save(); + await TitleName.destroy({ + where: { + title_id: episodeTitle.id, + } + }) + const primaryTitleName = await TitleName.build({ + region: null, + languages: null, + original: false, + name: imdbResult.primaryTitle, + }); + await primaryTitleName.setTitle(episodeTitle.id, {save: false}); + await primaryTitleName.save(); + const originalTitleName = await TitleName.build({ + region: null, + languages: null, + original: true, + name: imdbResult.originalTitle, + }); + await originalTitleName.setTitle(episodeTitle.id, {save: false}); + await originalTitleName.save(); + for (let el of tmdbTranslations.translations) { + const titleName = await TitleName.build({ + region: el.iso_3166_1, + languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], + original: false, + name: el.data.name, + }) + await titleName.setTitle(episodeTitle.id, {save: false}); + await titleName.save(); + } + await TitleDescription.destroy({ + where: { + title_id: episodeTitle.id, + } + }) + const originalTitleDescription = await TitleDescription.build({ + region: null, + languages: null, + overview: tmdbResult.overview, + tagline: tmdbResult.tagline, + }); + await originalTitleDescription.setTitle(episodeTitle.id, {save: false}); + await originalTitleDescription.save(); + for (let el of tmdbTranslations.translations) { + const titleDescription = await TitleDescription.build({ + region: el.iso_3166_1, + languages: el.iso_639_1 ? el.iso_639_1.split(",") : [], + overview: el.data.overview, + tagline: el.data.tagline, + }) + await titleDescription.setTitle(episodeTitle.id, {save: false}); + await titleDescription.save(); } + + return episodeTitle; } chooseImages(originalLanguage, tmdbImages, fanartImages) { @@ -115,12 +254,12 @@ class MetadataLoader { confidence: ranking_confidence(element.vote_average, element.vote_count), lang_quality: containsText ? ( element.iso_639_1 === originalLanguage ? 1.5 : - element.iso_639_1 === null ? 1 : - 0 + element.iso_639_1 === null ? 1 : + 0 ) : ( element.iso_639_1 === null ? 1.5 : - element.iso_639_1 === originalLanguage ? 1 : - 0 + element.iso_639_1 === originalLanguage ? 1 : + 0 ), megapixels: (element.height * element.width) / 1000000, ...element @@ -231,9 +370,64 @@ class MetadataLoader { } ].filter(el => el !== null); - await Promise.all(imageData.map(img => downloadFile(img.url, path.join(filePath, "metadata", img.type + path.extname(img.url))))) + return await Promise.all(imageData.map(async img => { + const headers = await downloadFile(img.url, path.join(filePath, "metadata", img.type + path.extname(img.url))); + return { + mime: headers["content-type"], + ...img, + } + })); + } - return imageData; + async processImageMetadata(title, images) { + await TitleImage.destroy({ + where: { + title_id: title.id, + } + }) + for (let image of images) { + const titleImage = await TitleImage.build({ + type: image.type, + mime: image.mime, + src: image.src, + }) + await titleImage.setTitle(title.id, {save: false}); + await titleImage.save(); + } + } + + async processMediaMetadata(title, media) { + await TitleMedia.destroy({ + where: { + title_id: title.id, + } + }) + for (let format of media.media) { + const titleMedia = await TitleMedia.build({ + mime: format.container, + codecs: [...new Set(format.tracks.flatMap(track => track.codecs))], + languages: [...new Set(format.tracks.map(track => track.language).filter(it => !!it))], + src: format.src, + }) + await titleMedia.setTitle(title.id, {save: false}); + await titleMedia.save(); + } + await TitleSubtitles.destroy({ + where: { + title_id: title.id, + } + }) + for (let subtitle of media.subtitles) { + const titleSubtitles = await TitleSubtitles.build({ + language: subtitle.language, + region: subtitle.region, + specifier: subtitle.specifier, + format: subtitle.format, + src: subtitle.src + }) + await titleSubtitles.setTitle(title.id, {save: false}); + await titleSubtitles.save(); + } } async loadEpisodeMetadata(ids, episodeIdentifier) { @@ -249,13 +443,11 @@ class MetadataLoader { ...tmdbSources.map(url => this.tmdb.request(url).catch(_ => null)), ].filter(el => el !== null)); - const metadata = this.transformEpisodeData(ids, imdbResult, tmdbResult, tmdbTranslations); - if (!metadata) { - return null; - } + const title = await this.transformEpisodeData(ids, episodeIdentifier, imdbResult, tmdbResult, tmdbTranslations); + if (!title) return; return { - metadata: metadata, - images: this.chooseImages(metadata.originalLanguage, tmdbImages), + title: title, + images: this.chooseImages(title.original_language, tmdbImages), }; } @@ -280,10 +472,10 @@ class MetadataLoader { this.fanart.request(fanartSource).catch(_ => null) // do nothing, it just means it wasn’t found ].filter(el => el !== null)); - const metadata = this.transformData(ids, imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations); + const title = await this.transformData(ids, imdbResult, tmdbResult, tmdbContentRatings, tmdbTranslations); return { - metadata: metadata, - images: this.chooseImages(metadata.originalLanguage, tmdbImages, fanartImages), + title: title, + images: this.chooseImages(title.original_language, tmdbImages, fanartImages) }; } } diff --git a/src/model.js b/src/model.js new file mode 100644 index 0000000..7fd8a13 --- /dev/null +++ b/src/model.js @@ -0,0 +1,37 @@ +import sequelize from "sequelize"; + +export class Genre extends sequelize.Model { +} + +export class Person extends sequelize.Model { +} + +export class Title extends sequelize.Model { +} + +export class TitleCast extends sequelize.Model { +} + +export class TitleDescription extends sequelize.Model { +} + +export class TitleEpisode extends sequelize.Model { +} + +export class TitleGenre extends sequelize.Model { +} + +export class TitleImage extends sequelize.Model { +} + +export class TitleMedia extends sequelize.Model { +} + +export class TitleName extends sequelize.Model { +} + +export class TitleRating extends sequelize.Model { +} + +export class TitleSubtitles extends sequelize.Model { +} diff --git a/src/process_content.js b/src/process_content.js new file mode 100644 index 0000000..ba62138 --- /dev/null +++ b/src/process_content.js @@ -0,0 +1,61 @@ +import path from "path"; +import {promises as fsPromises} from "fs"; + +async function processContent(basePath, fileManager, loader) { + async function processMovie(filePath) { + const {name, year} = /^(?<name>.+) \((?<year>\d+)\)$/.exec(path.basename(filePath)).groups; + + const ids = (await fileManager.findIds(filePath)) || (await loader.identifyMovie(name, year)); + if (!ids) { + console.error(`Could not identify movie ${name} (${year}) at ${filePath}`) + return; + } + const [media, {title, images}] = await Promise.all([ + fileManager.findMedia(filePath), + loader.loadMetadata(ids), + ]); + const imageData = await loader.processImages(basePath, filePath, images); + await loader.processImageMetadata(title, imageData); +await loader.processMediaMetadata(title, media); + + await fsPromises.writeFile(path.join(filePath, "ids.json"), JSON.stringify(ids, null, 2)); + } + + async function processEpisode(showIds, episodeIdentifier, filePath) { + const [media, {title, images}] = await Promise.all([ + fileManager.findMedia(filePath), + loader.loadEpisodeMetadata(showIds, episodeIdentifier), + ]); + const imageData = await loader.processImages(basePath, filePath, images); + await loader.processImageMetadata(title, imageData); + await loader.processMediaMetadata(title, media); + } + + async function processShow(filePath) { + const {name, year} = /^(?<name>.+) \((?<year>\d+)\)$/.exec(path.basename(filePath)).groups; + const ids = (await fileManager.findIds(filePath)) || (await loader.identifyShow(name, year)); + if (!ids) { + console.error(`Could not identify show ${name} (${year}) at ${filePath}`) + return; + } + const episodes = await fileManager.listEpisodes(filePath); + + const {title, images} = await loader.loadMetadata(ids); + const imageData = await loader.processImages(basePath, filePath, images); + await loader.processImageMetadata(title, imageData); + await Promise.all([ + ...episodes.map(async ({episodeIdentifier, filePath}) => await processEpisode(ids, episodeIdentifier, filePath).catch(err => { + console.error(`Error processing episode ${JSON.stringify(episodeIdentifier)} of show ${JSON.stringify(ids)}: `, err); + })), + fsPromises.writeFile(path.join(filePath, "ids.json"), JSON.stringify(ids, null, 2)), + ]); + } + + const [movies, shows] = await Promise.all([fileManager.listMovies(), fileManager.listShows()]); + await Promise.all([ + ...movies.map(processMovie), + ...shows.map(processShow) + ]); +} + +export default processContent; \ No newline at end of file diff --git a/src/storage.js b/src/storage.js new file mode 100644 index 0000000..b2eb8c5 --- /dev/null +++ b/src/storage.js @@ -0,0 +1,287 @@ +import sequelize from 'sequelize'; +import { + Genre, + Person, + Title, + TitleCast, + TitleDescription, + TitleEpisode, + TitleGenre, + TitleImage, + TitleMedia, + TitleName, + TitleRating, + TitleSubtitles +} from "./model"; + +class Backend { + /** + * @type Sequelize + */ + db; + + constructor(db) { + this.db = db; + + Genre.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + tmdb_id: sequelize.DataTypes.INTEGER, + name: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'genre', + indexes: [] + }); + Person.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + imdb_id: sequelize.DataTypes.STRING(64), + name: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'people', + indexes: [] + }); + Title.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + imdb_id: sequelize.DataTypes.STRING(64), + tmdb_id: sequelize.DataTypes.INTEGER, + tvdb_id: sequelize.DataTypes.INTEGER, + original_language: sequelize.DataTypes.STRING(32), + runtime: sequelize.DataTypes.INTEGER, + year_start: sequelize.DataTypes.INTEGER, + year_end: sequelize.DataTypes.INTEGER, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title', + indexes: [] + }); + TitleCast.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + category: sequelize.DataTypes.TEXT, + characters: sequelize.DataTypes.ARRAY(sequelize.DataTypes.TEXT), + job: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_cast', + indexes: [] + }); + TitleCast.belongsTo(Title); + Title.hasMany(TitleCast); + TitleCast.belongsTo(Person); + Person.hasMany(TitleCast); + + TitleDescription.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + region: sequelize.DataTypes.STRING(32), + languages: sequelize.DataTypes.ARRAY(sequelize.DataTypes.STRING(32)), + overview: sequelize.DataTypes.TEXT, + tagline: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_description', + indexes: [] + }); + TitleDescription.belongsTo(Title); + Title.hasMany(TitleDescription); + + TitleEpisode.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + season_number: sequelize.DataTypes.STRING(64), + episode_number: sequelize.DataTypes.STRING(64), + air_date: sequelize.DataTypes.DATEONLY, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_episode', + indexes: [ + { + fields: [ + 'show_id', + { + attribute: 'season_number', + collate: 'C', + order: 'ASC', + }, + { + attribute: 'episode_number', + collate: 'C', + order: 'ASC', + } + ] + }, + { + using: 'BTREE', + fields: [ + 'show_id', + { + attribute: 'air_date', + order: 'ASC', + } + ] + } + ] + }); + TitleEpisode.belongsTo(Title, {as: "Show", foreignKey: "show_id"}); + TitleEpisode.belongsTo(Title, {as: "Episode", foreignKey: "episode_id"}); + Title.hasMany(TitleEpisode, { foreignKey: "show_id", as: 'Episodes'}); + Title.hasOne(TitleEpisode, { foreignKey: "episode_id", as: 'Show'}); + TitleGenre.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + title_id: sequelize.DataTypes.UUID, + genre_id: sequelize.DataTypes.UUID, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_genre', + indexes: [] + }); + TitleGenre.belongsTo(Title); + Title.hasMany(TitleGenre); + TitleGenre.belongsTo(Genre); + Genre.hasMany(TitleGenre); + TitleImage.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + type: sequelize.DataTypes.STRING(64), + mime: sequelize.DataTypes.TEXT, + src: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_image', + indexes: [] + }); + TitleImage.belongsTo(Title); + Title.hasMany(TitleImage); + TitleMedia.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + mime: sequelize.DataTypes.STRING(64), + codecs: sequelize.DataTypes.ARRAY(sequelize.DataTypes.STRING(64)), + languages: sequelize.DataTypes.ARRAY(sequelize.DataTypes.STRING(32)), + src: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_media', + indexes: [] + }); + TitleMedia.belongsTo(Title); + Title.hasMany(TitleMedia); + TitleName.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + region: sequelize.DataTypes.STRING(32), + languages: sequelize.DataTypes.ARRAY(sequelize.DataTypes.STRING(32)), + original: sequelize.DataTypes.BOOLEAN, + name: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_name', + indexes: [] + }); + TitleName.belongsTo(Title); + Title.hasMany(TitleName); + TitleRating.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + region: sequelize.DataTypes.STRING(32), + certification: sequelize.DataTypes.STRING(32), + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_rating', + indexes: [] + }); + TitleRating.belongsTo(Title); + Title.hasMany(TitleRating); + TitleSubtitles.init({ + id: { + type: sequelize.DataTypes.UUID, + defaultValue: sequelize.DataTypes.UUIDV4, + allowNull: false, + primaryKey: true + }, + format: sequelize.DataTypes.STRING(64), + language: sequelize.DataTypes.STRING(64), + region: sequelize.DataTypes.STRING(64), + specifier: sequelize.DataTypes.STRING(64), + src: sequelize.DataTypes.TEXT, + }, { + sequelize: this.db, + underscored: true, + modelName: 'title_subtitles', + indexes: [] + }); + TitleSubtitles.belongsTo(Title); + Title.hasMany(TitleSubtitles); + } + + async sync() { + await Promise.all([ + Genre.sync(), Person.sync(), Title.sync(), + TitleCast.sync(), TitleDescription.sync(), TitleEpisode.sync(), TitleGenre.sync(), TitleImage.sync(), + TitleMedia.sync(), TitleName.sync(), TitleRating.sync(), TitleSubtitles.sync(), + ]); + } + +} + +export default Backend; \ No newline at end of file diff --git a/src/util/download-file.js b/src/util/download-file.js index 5207943..58555b8 100644 --- a/src/util/download-file.js +++ b/src/util/download-file.js @@ -9,7 +9,7 @@ function downloadFile(url, filePath) { https.get(url, function (response) { response.pipe(file); file.on('close', function () { - resolve(); + resolve(response.headers); }) file.on('finish', function () { file.close() diff --git a/src/util/video-mime.js b/src/util/video-mime.js index 43b3edb..4526baa 100644 --- a/src/util/video-mime.js +++ b/src/util/video-mime.js @@ -23,7 +23,7 @@ class VideoMimeParser { return { id: i, type: track.mimeType.substr(0, track.mimeType.indexOf('/')), - codec: [...new Set(representations.map(r => r.codecs))].join(","), + codecs: [...new Set(representations.map(r => r.codecs))], bitrate: representations.map(r => r.bandwidth).sort()[0], language: track.lang } @@ -53,15 +53,18 @@ class VideoMimeParser { async parseMediaInfoMp4(filePath) { const [stdout] = await promisify(exec, `${this.mp4boxPath} --format json "${filePath}"`); const info = JSON.parse(stdout.replace(/,\s*([\]}])/g, "$1")); + const audioTrack = info.tracks.filter(track => track.type.toLowerCase() === "audio")[0]; + const videoTrack = info.tracks.filter(track => track.type.toLowerCase() === "video")[0]; return { container: "video/mp4", duration: +info.movie.duration, - tracks: info.tracks.filter(track => ["audio", "video"].includes(track.type.toLowerCase())).map(track => { + tracks: [audioTrack, videoTrack].map(track => { return { id: track.id, type: track.type.toLowerCase(), codecs: [ - track.sample_descriptions[0].codecs_string + track.sample_descriptions[0].codecs_string || + track.sample_descriptions[0].coding ], language: track.language === "und" ? null : track.language, } -- GitLab