Automatic import from remote drive (#86)

* Add initial files for handling automatic importing from seedboxes

* Initial structure and storage driver integration

* Add method to find files recursively

* Remove . from file extensions

* Add sample config for seedboxes

* Filter which movies need to be import from remote storage

* Basic importing of movies from remote storage

* Add support for importing episodes

* Update package-lock.json

* Update package-lock.json

* Add rest endpoint to import media from remote storage

* Only import matched episodes and store episodes in series directories

* Add option to disable an auto importer config

Author: robinp7720

package-lock.json

@@ -34,6 +34,7 @@
     "fluent-ffmpeg": "^2.1.2",
     "guessit-exec": "^0.0.1",
     "guessit-wrapper": "^1.0.1",
+    "jsftp": "^2.1.3",
     "jsonwebtoken": "^8.5.1",
     "mime-types": "^2.1.30",
     "mkdirp": "^1.0.4",
@@ -47,7 +48,7 @@
     "restify-cors-middleware": "^1.1.1",
     "restify-errors": "^8.0.2",
     "sequelize": "^6.6.2",
-    "sharp": "^0.28.1",
+    "sharp": "^0.29.1",
     "socket.io": "^2.3.0",
     "sqlite3": "^5.0.2",
     "uuid": "^8.3.2",
@@ -61,9 +62,9 @@
     "@types/bcrypt": "^3.0.1",
     "@types/restify-errors": "^4.3.3",
     "@types/socket.io": "^2.1.13",
+    "chai": "^4.3.3",
     "eslint": "^7.25.0",
     "eslint-plugin-jsdoc": "^32.3.3",
-    "chai": "^4.3.3",
     "expect.js": "^0.3.1",
     "jshint": "^2.12.0",
     "mocha": "^8.3.2",

package.json

@@ -81,12 +81,12 @@
   },
   "fileExtensions": {
     "video": [
-      ".mp4",
-      ".avi",
-      ".iso",
-      ".m4v",
-      ".mkv",
-      ".mk3d"
+      "mp4",
+      "avi",
+      "iso",
+      "m4v",
+      "mkv",
+      "mk3d"
     ]
   },
   "authentication": {
@@ -126,5 +126,24 @@
         "key": "/etc/oblecto/keys/tria.pub"
       }
     }
-  }
+  },
+  "seedboxes": [
+    {
+      "name": "Main seedbox",
+      "storageDriver": "ftp",
+      "storageDriverOptions": {
+        "host": "",
+        "port": 21,
+        "username": "username",
+        "password": "password"
+      },
+      "mediaImport": {
+        "movieDirectory": "/downloads/finished/movie/",
+        "seriesDirectory": "/downloads/finished/series/"
+      },
+      "automaticImport": true,
+      "deleteOnImport": false,
+      "enabled": false
+    }
+  ]
 }

res/config.json

@@ -1,5 +1,6 @@
 import { Movie } from '../../../models/movie';
 import { fileExists } from '../../../submodules/utils';
+import logger from '../../../submodules/logger';
 
 /**
  * @typedef {import('../../oblecto').default} Oblecto
@@ -20,7 +21,7 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectArtworkMovieFanart(movie) {
-        if (!await fileExists(this.oblecto.artworkUtils.movieFanartPath(movie))) return;
+        if (await fileExists(this.oblecto.artworkUtils.movieFanartPath(movie))) return;
 
         this.oblecto.queue.queueJob('downloadMovieFanart', movie);
     }
@@ -31,7 +32,7 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectArtworkMoviePoster(movie) {
-        if (!await fileExists(this.oblecto.artworkUtils.moviePosterPath(movie))) return;
+        if (await fileExists(this.oblecto.artworkUtils.moviePosterPath(movie))) return;
 
         this.oblecto.queue.queueJob('downloadMoviePoster', movie);
     }
@@ -41,6 +42,8 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectAllMovieFanart() {
+        logger.log('DEBUG', 'Collecting Movie fanart to download');
+
         let movies = await Movie.findAll();
 
         for (let movie of movies) {
@@ -53,6 +56,8 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectAllMoviePosters() {
+        logger.log('DEBUG', 'Collecting Movie posters to download');
+
         let movies = await Movie.findAll();
 
         for (let movie of movies) {

src/lib/artwork/movies/MovieArtworkCollector.js

@@ -20,6 +20,11 @@ export default class FileIndexer {
         this.oblecto.queue.registerJob('indexFileStreams', this.indexVideoFileStreams);
     }
 
+    /**
+     *
+     * @param videoPath
+     * @return {File}
+     */
     async indexVideoFile(videoPath) {
         let parsedPath = Path.parse(videoPath);
 

src/lib/indexers/files/FileIndexer.js

@@ -33,7 +33,7 @@ export default class MovieCollector {
      * @returns {Promise<void>}
      */
     async collectFile(file) {
-        let extension = path.parse(file).ext.toLowerCase();
+        let extension = path.parse(file).ext.toLowerCase().replace('.', '');
 
         if (this.oblecto.config.fileExtensions.video.indexOf(extension) !== -1) {
             this.oblecto.queue.queueJob('indexMovie',{ path: file });

src/lib/indexers/movies/MovieCollector.js

@@ -1,5 +1,5 @@
 import recursive from 'recursive-readdir';
-import path from 'path';
+import { extname } from 'path';
 
 /**
  * @typedef {import('../../oblecto').default} Oblecto
@@ -41,7 +41,7 @@ export default class SeriesCollector {
      * @returns {Promise<void>}
      */
     async collectFile(file) {
-        let extension = path.parse(file).ext.toLowerCase();
+        let extension = extname(file).toLowerCase().replace('.','');
 
         if (this.oblecto.config.fileExtensions.video.indexOf(extension) !== -1) {
             this.oblecto.queue.queueJob('indexEpisode',{ path: file });

src/lib/indexers/series/SeriesCollector.js

@@ -1,4 +1,4 @@
-import { Op } from 'sequelize';
+import {Op} from 'sequelize';
 
 import AggregateIdentifier from '../../common/AggregateIdentifier';
 
@@ -7,16 +7,17 @@ import TmdbEpisodeIdentifier from './identifiers/TmdbEpisodeIdentifier';
 import TvdbSeriesIdentifier from './identifiers/TvdbSeriesIdentifier';
 import TvdbEpisodeIdentifier from './identifiers/TvdbEpisodeIdentifier';
 
-import { Series } from '../../../models/series';
-import { Episode } from '../../../models/episode';
-import { File } from '../../../models/file';
+import {Series} from '../../../models/series';
+import {Episode} from '../../../models/episode';
+import {File} from '../../../models/file';
 
 import IdentificationError from '../../errors/IdentificationError';
 import logger from '../../../submodules/logger';
 import guessit from '../../../submodules/guessit';
 
 /**
  * @typedef {import('../../oblecto').default} Oblecto
+ * @typedef {import('../../../submodules/guessit').GuessitIdentification} GuessitIdentification
  */
 
 /**
@@ -98,6 +99,14 @@ export default class SeriesIndexer {
         return series;
     }
 
+    async identify(episodePath) {
+        const guessitIdentification = await guessit.identify(episodePath);
+        const seriesIdentification = await this.seriesIdentifier.identify(episodePath, guessitIdentification);
+        const episodeIdentification = await this.episodeIdentifer.identify(episodePath, guessitIdentification, seriesIdentification);
+
+        return { ...seriesIdentification, ...episodeIdentification };
+    }
+
     /**
      * Index a specific file and identify it as a series
      *
@@ -112,8 +121,6 @@ export default class SeriesIndexer {
          */
         const guessitIdentification = await guessit.identify(episodePath);
 
-        console.log(guessitIdentification.source);
-
         // Some single season shows usually don't have a season in the title,
         // therefore whe should set it to 1 by default.
         if (!guessitIdentification.season) {

src/lib/indexers/series/SeriesIndexer.js

@@ -42,6 +42,7 @@ import FileIndexer from '../indexers/files/FileIndexer';
 
 import { initDatabase } from '../../submodules/database';
 import StreamSessionController from '../streamSessions/StreamSessionController';
+import SeedboxController from '../seedbox/SeedboxController';
 
 export default class Oblecto {
     constructor(config) {
@@ -88,6 +89,9 @@ export default class Oblecto {
 
         this.streamSessionController = new StreamSessionController(this);
 
+        this.seedboxController = new SeedboxController(this);
+        this.seedboxController.loadAllSeedboxes();
+
         if (config.federation.enable) {
             this.fedartionController = new FederationController(this);
             this.federationClientController = new FederationClientController(this);

src/lib/oblecto/index.js

@@ -13,7 +13,7 @@ export default class Queue {
             if (!this.jobs[job.id]) return callback();
 
             let jobTimeout = setTimeout(() => {
-                logger.log('WARN', `Job ${job.id} is taking a long time. Maybe something is wrong?`, JSON.stringify(job));
+                logger.log('DEBUG', `Job ${job.id} is taking a long time. Maybe something is wrong?`, JSON.stringify(job));
             }, 20000);
 
             this.jobs[job.id](job.attr)