'use strict';

/**
 * Configuration dependencies.
 */

var config = require('../config/production/config');
Object.keys(config).length === 0 &&
  (config = require('../config/production/config.backup'));
var modules = require('../config/production/modules');
Object.keys(modules).length === 0 &&
  (modules = require('../config/production/modules.backup'));

/**
 * Module dependencies.
 */

var CP_get = require('./CP_get');
var CP_save = require('./CP_save');

/**
 * Node dependencies.
 */

var fs = require('fs');
var path = require('path');
var os = require('os-utils');
var op = require('object-path');
var async = require('async');
var adop = require('adop');
var axios = require('axios');
var convert = require('xml-js');
var htmlmetaparser = require('htmlmetaparser');
var countries = require('i18n-iso-countries');
var htmlparser2 = require('htmlparser2');
var LRU = require('lru-cache');
var cache = new LRU({ max: 1000 });
var HttpsProxyAgent = require('https-proxy-agent');
var sphinx = require('sphinx');
var axiosCookieJarSupport = require('axios-cookiejar-support').default;
var tough = require('tough-cookie');
var Cookie = tough.Cookie;

axiosCookieJarSupport(axios);

var cookieJar = new tough.CookieJar();

if (config.movies.cookies) {
  try {
    var cookies = JSON.parse(config.movies.cookies);
    Object.keys(cookies).forEach(function(domain) {
      cookies[domain].forEach(function(cookie) {
        if (!cookie) return;
        if (cookie.name) {
          cookie.key = cookie.name;
        }
        if (cookie.expirationDate) {
          cookie.expires = new Date(parseInt(cookie.expirationDate) * 1000);
        }
        cookieJar.setCookie(Cookie.fromJSON(cookie), domain, function(e, c) {});
      });
    });
  } catch (e) {
    cookieJar = undefined;
    console.error(e);
  }
}

/**
 * Global env.
 */

try {
  var p = tryParseJSON(
    fs.readFileSync(
      path.join(path.dirname(__filename), '..', 'process.json'),
      'utf8'
    )
  );
  var e = p.apps[0].env;
  for (var prop in e) {
    if (e.hasOwnProperty(prop)) {
      process.env[prop] = e[prop];
    }
  }
} catch (err) {
  return console.log('NOT FILE PROCESS DATA');
}

process.env['NO_CACHE'] = true;

var run = process.argv && typeof process.argv[2] !== 'undefined';
var timeZone = new Date();
var hour = new Date(timeZone).getHours() + 1;

console.time('[REALTIME] DONE');

console.log(
  '[REALTIME]',
  new Date()
    .toJSON()
    .replace('T', ' ')
    .split('.')[0],
  '[',
  Math.ceil(os.freemem()),
  'MB ]',
  os.loadavg(1).toFixed(2),
  os.loadavg(5).toFixed(2),
  os.loadavg(15).toFixed(2)
);

if (Math.ceil(os.freemem()) < 50) {
  console.log('The server is overloaded to start get movies.');
  return process.exit(0);
}

/**
 * Get movie information.
 *
 */

(function getInfo() {
  if (!config.movies.cron) return process.exit();

  var tasks = [];
  config.movies.cron.forEach(function(task) {
    var parse = task
      .replace(/(^\s*)|(\s*)$/g, '')
      .replace(/\s*~\s*/g, '~')
      .split('~');
    if (task.charAt(0) === '#') return;
    tasks.push({
      hour: parse[0] && parseInt(parse[0]) ? parseInt(parse[0]) : 0,
      page: parse[1] || '',
      path: (parse[2] || '').split('<>')[0].trim(),
      id: parse[3] || '',
      info: parse.slice(4),
      type_id: ((parse[2] || '').split('<>')[1] || '').trim(),
      ids: []
    });
  });

  var update_m = 0;
  var added_m = 0;

  async.eachOfLimit(
    tasks,
    1,
    function(task, task_index, callback) {
      if (task.hour && task.hour % hour) {
        return callback();
      }
      if (task.hour === 0 && !run) {
        return callback();
      }
      var i = 1;
      var fail_req = 5;
      var next_req = 1;
      var prev_req = 0;
      var some_req = 0;
      var retry_req = '';
      var iter_req = 0;
      var ids = cache.get(task.page + task.path + task.type_id) || [];
      if (ids && ids.length) {
        fail_req = 0;
        next_req = 0;
      }
      var url_next = '';
      var url_next_path = '';
      async.forever(
        function(next) {
          if (
            task.page === '' ||
            task.page === 'realtime' ||
            task.page === 'rt' ||
            task.page === 'database' ||
            task.page === 'db'
          ) {
            CP_get.movies(
              { certainly: true },
              100,
              'kinopoisk-id-up',
              i,
              false,
              function(err, rt) {
                if (ids.length && !(ids.length % 1000)) {
                  console.log('[REALTIME]', ids.length);
                }
                if (err) {
                  console.error(err);
                  return next('STOP');
                }
                if (rt && rt.length) {
                  rt.forEach(function(r) {
                    var id = '';
                    if (task.path.indexOf('custom.') + 1) {
                      var cst = {};
                      try {
                        cst = r.custom ? JSON.parse(r.custom) : {};
                      } catch (e) {}
                      id = op.get({ custom: cst }, task.path);
                      if (id) {
                        task.type_id = task.path;
                      }
                    } else {
                      id = r['kp_id'];
                      task.type_id = 'kp_id';
                    }
                    if (id && ids.indexOf(id + '') === -1) {
                      ids.push(id + '');
                    }
                  });
                  iter_req = iter_req + ids.length;
                  ids_iteration(function() {
                    some_req = 0;
                    ids = [];
                    i++;
                    return next();
                  });
                } else {
                  return next('STOP');
                }
              }
            );
          } else {
            if (task.page.indexOf('[page]') === -1 && i > 1) {
              console.log('[PAGE] 1');
              return next('STOP');
            }
            if (fail_req <= 0) {
              if (next_req) {
                i++;
                fail_req = 5;
                next_req = 0;
                retry_req = '';
                return next();
              }
              console.log('[FAIL] REQUEST');
              return next('STOP');
            }
            if (some_req >= 5) {
              console.log('[SOME] LENGTH');
              return next('STOP');
            }
            if (prev_req === ids.length) {
              some_req++;
            } else {
              retry_req = '';
              some_req = 0;
              prev_req = 0;
            }
            prev_req = ids.length;
            var url_req = task.page.replace('[page]', i);
            var re = /\[page]\[(.+)]/i;
            if (re.test(task.page) && !url_next_path && !url_next) {
              url_req = task.page.replace(/[a-z0-9]+=\[page]\[(.+)]/i, '');
              var url_next_path_exec = re.exec(task.page);
              if (
                url_next_path_exec &&
                url_next_path_exec.length &&
                url_next_path_exec[1]
              ) {
                url_next_path = url_next_path_exec[1];
              }
            }
            if (url_next) {
              url_req = url_next;
            }
            if (retry_req) {
              url_req =
                retry_req +
                (retry_req.indexOf('?') + 1 ? '&' : '?') +
                'clear-cache-request=' +
                Math.random()
                  .toString(36)
                  .slice(2);
            } else {
              console.log('[REALTIME]', ids.length, url_req);
            }
            var opt = {
              headers: {
                'User-Agent':
                  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0'
              }
            };
            if (config.movies.cookies && cookieJar) {
              opt.jar = cookieJar;
              opt.withCredentials = true;
            }
            var hostname = require('url').parse(url_req).hostname;
            var movies_proxy = '';
            if (config.movies.proxy && config.movies.proxy.length) {
              var proxies = config.movies.proxy.filter(function(p) {
                var only_proxy = p.split('<>')[0].trim();
                var only_domain = (p.split('<>')[1] || '').trim();
                if (only_domain) {
                  if (only_proxy && only_domain === hostname) {
                    return true;
                  }
                } else if (only_proxy) {
                  return true;
                }
                return false;
              });
              var proxy =
                proxies && proxies.length
                  ? proxies[Math.floor(Math.random() * proxies.length)]
                  : '';
              if (proxy) {
                var proxy_obj = {};
                if (proxy.indexOf('@') + 1) {
                  var host_auth = proxy.split('@');
                  proxy_obj['host'] = host_auth[1].split(':')[0];
                  proxy_obj['port'] = parseInt(host_auth[1].split(':')[1]);
                  proxy_obj['auth'] = host_auth[0];
                } else {
                  proxy_obj['host'] = proxy.split(':')[0];
                  proxy_obj['port'] = parseInt(proxy.split(':')[1]);
                }
                opt.httpsAgent = new HttpsProxyAgent(proxy_obj);
                movies_proxy =
                  proxy.indexOf('@') + 1 ? proxy.split('@')[1] : proxy;
              }
            }
            axios(url_req, opt)
              .then(function(r) {
                if (!r || !r.data) {
                  if (retry_req) {
                    i++;
                    retry_req = '';
                  } else {
                    retry_req = url_req;
                  }
                  console.error(
                    '[REALTIME]',
                    'STOP PAGE (' + (6 - fail_req) + ' fails)',
                    url_req,
                    'PROXY',
                    movies_proxy || false
                  );
                  fail_req = fail_req - 1;
                  setTimeout(function() {
                    return next();
                  }, 5000);
                } else if (r.data) {
                  if (
                    typeof r.data === 'string' &&
                    r.data.indexOf('<html') + 1
                  ) {
                    if (task.path) {
                      var ids_list = r.data.match(task.path.split('<>')[0]);
                      if (ids_list && ids_list.length) {
                        ids_list.forEach(function(id) {
                          id = id.trim();
                          if (id && ids.indexOf(id) === -1) {
                            ids.push(id);
                          }
                        });
                      }
                    }
                    i++;
                    return next();
                  } else {
                    var r_data = tryParseJSON(r.data);
                    if (r_data && typeof r_data === 'object' && url_next_path) {
                      url_next = op.get(r_data, url_next_path);
                    }
                    if (r_data && (!task.id || task.id === '[url]')) {
                      if (!task.path || task.id === '[url]') {
                        if (
                          typeof r_data === 'object' &&
                          Array.isArray(r_data)
                        ) {
                          ids = r_data;
                          iter_req = iter_req + ids.length;
                          ids_iteration(function() {
                            some_req = 0;
                            ids = [];
                            i++;
                            return next();
                          });
                        } else {
                          i++;
                          return next();
                        }
                      } else {
                        var get_array = op.get(r_data, task.path);
                        if (
                          typeof get_array === 'object' &&
                          Array.isArray(get_array)
                        ) {
                          ids = get_array;
                          iter_req = iter_req + ids.length;
                          ids_iteration(function() {
                            some_req = 0;
                            ids = [];
                            i++;
                            return next();
                          });
                        } else {
                          i++;
                          return next();
                        }
                      }
                    } else {
                      var all = adop(r_data, [
                        {
                          name: 'id',
                          path: task.path.split('<>')[0],
                          type: task.path.split('<>')[1] || undefined,
                          regex: task.path.split('<>')[2] || undefined
                        }
                      ]);
                      if (all && all.length) {
                        all.forEach(function(a) {
                          if (
                            a &&
                            a.id &&
                            a.id !== 'null' &&
                            a.id !== 'false' &&
                            a.id !== 'undefined' &&
                            a.id !== 'n/a' &&
                            a.id !== 'N/A' &&
                            ids.indexOf(a.id) === -1
                          ) {
                            ids.push(a.id);
                          }
                        });
                        if (!ids.length) {
                          some_req = some_req + 1;
                          ids = [];
                          i++;
                          return next();
                        }
                        iter_req = iter_req + ids.length;
                        ids_iteration(function() {
                          some_req = 0;
                          ids = [];
                          i++;
                          return next();
                        });
                      } else {
                        if (retry_req) {
                          i++;
                          retry_req = '';
                        } else {
                          retry_req = url_req;
                        }
                        console.error(
                          '[REALTIME]',
                          'STOP PAGE (' + (6 - fail_req) + ' fails)',
                          url_req,
                          'PROXY',
                          movies_proxy || false
                        );
                        fail_req = fail_req - 1;
                        setTimeout(function() {
                          return next();
                        }, 5000);
                      }
                    }
                  }
                } else {
                  return next();
                }
              })
              .catch(function(err) {
                if (retry_req) {
                  i++;
                  retry_req = '';
                } else {
                  retry_req = url_req;
                }
                console.error(
                  '[REALTIME]',
                  'STOP PAGE (' + (6 - fail_req) + ' fails)',
                  url_req,
                  err.response && err.response.status + '',
                  err.response && err.response.statusText + '',
                  'PROXY',
                  movies_proxy || false
                );
                fail_req = fail_req - 1;
                setTimeout(function() {
                  return next();
                }, 5000);
              });
          }
        },
        function() {
          ids_iteration(function() {
            callback();
          });
        }
      );
      function ids_iteration(callback) {
        var pool = sphinx.createPool({});
        cache.set(task.page + task.path + task.type_id, ids);
        async.eachOfLimit(
          ids,
          5,
          function(id, id_index, callback) {
            if (
              task.id &&
              !/\[[a-z0-9_]+?]/i.test(task.id + '') &&
              id_index >= 1
            ) {
              if (
                !(
                  task.page === '' ||
                  task.page === 'realtime' ||
                  task.page === 'rt' ||
                  task.page === 'database' ||
                  task.page === 'db'
                )
              ) {
                return callback();
              }
            }
            var task_type_id = task.type_id || '';
            var task_url =
              typeof id === 'object'
                ? ''
                : (task.id + '').replace(/\[[a-z0-9_]+?]/i, id + '');
            if (task_url) {
              var opt = {
                headers: {
                  'User-Agent':
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0'
                }
              };
              if (config.movies.cookies && cookieJar) {
                opt.jar = cookieJar;
                opt.withCredentials = true;
              }
              var hostname = require('url').parse(task_url).hostname;
              var movies_proxy = '';
              var movies_captcha = '';
              if (config.movies.proxy && config.movies.proxy.length) {
                var proxies = config.movies.proxy.filter(function(p) {
                  var only_proxy = p.split('<>')[0].trim();
                  var only_domain = (p.split('<>')[1] || '').trim();
                  if (only_domain) {
                    if (only_proxy && only_domain === hostname) {
                      return true;
                    }
                  } else if (only_proxy) {
                    return true;
                  }
                  return false;
                });
                var proxy =
                  proxies && proxies.length
                    ? proxies[Math.floor(Math.random() * proxies.length)]
                    : '';
                if (proxy) {
                  var proxy_obj = {};
                  if (proxy.indexOf('@') + 1) {
                    var host_auth = proxy.split('@');
                    proxy_obj['host'] = host_auth[1].split(':')[0];
                    proxy_obj['port'] = parseInt(host_auth[1].split(':')[1]);
                    proxy_obj['auth'] = host_auth[0];
                  } else {
                    proxy_obj['host'] = proxy.split(':')[0];
                    proxy_obj['port'] = parseInt(proxy.split(':')[1]);
                  }
                  opt.httpsAgent = new HttpsProxyAgent(proxy_obj);
                  movies_proxy =
                    proxy.indexOf('@') + 1 ? proxy.split('@')[1] : proxy;
                }
              }
              axios(task_url, opt)
                .then(function(j) {
                  if (!j || !j.data) {
                    return callback();
                  }
                  if (
                    typeof j.data === 'string' &&
                    j.data.indexOf('<html') + 1
                  ) {
                    var handler = new htmlmetaparser.Handler(
                      function(err, json) {
                        if (!json || typeof json !== 'object') {
                          return callback();
                        }
                        if (
                          json &&
                          json.html &&
                          json.html.title &&
                          json.html.title === 'Ой!'
                        ) {
                          movies_captcha = 'CAPTCHA';
                          json = {};
                        }
                        movie_data(json, function() {
                          return callback();
                        });
                      },
                      { url: task_url }
                    );
                    var parser = new htmlparser2.Parser(handler, {
                      decodeEntities: true
                    });
                    parser.write(j.data);
                    parser.done();
                  } else {
                    var json = tryParseJSON(j.data);
                    if (!json || typeof json !== 'object') {
                      return callback();
                    }
                    if (task.id === '[url]') {
                      var movies_on_page = op.get(json, task.path);
                      if (Array.isArray(movies_on_page)) {
                        async.eachOfLimit(
                          movies_on_page,
                          5,
                          function(movie, id_index, callback) {
                            movie_data(movie, function() {
                              return callback();
                            });
                          },
                          function() {
                            return callback();
                          }
                        );
                      } else {
                        return callback();
                      }
                    } else {
                      movie_data(json, function() {
                        return callback();
                      });
                    }
                  }
                })
                .catch(function(err) {
                  console.log(
                    '[REALTIME]',
                    id_index + 1,
                    '/',
                    ids.length,
                    iter_req ? '[' + iter_req + ']' : '',
                    'ERROR MOVIE',
                    task_url,
                    err.response && err.response.status + '',
                    err.response && err.response.statusText + '',
                    'PROXY',
                    movies_proxy || false
                  );
                  return callback();
                });
            } else if (typeof id === 'object') {
              movie_data(id, function() {
                return callback();
              });
            } else {
              movie_data({}, function() {
                return callback();
              });
            }
            function movie_data(json, callback) {
              var movie = {};
              if (task_type_id) {
                if (task_type_id === 'id' || task_type_id === 'kp_id') {
                  movie['kp_id'] = id.replace(/[^0-9]/g, '');
                } else if (task_type_id.indexOf('custom.') + 1) {
                  op.set(movie, task_type_id, id.replace(/[^0-9]/g, ''));
                }
              }
              task.info.forEach(function(info) {
                var parse = info
                  .replace(/(^\s*)|(\s*)$/g, '')
                  .replace(/\s*<>\s*/g, '<>')
                  .split('<>');
                var sup_double = !!(parse[3] && parse[3].indexOf('==') + 1);
                var sup_parse = parse[3]
                  ? parse[3].indexOf('==') + 1
                    ? parse[3]
                        .replace(/(^\s*)|(\s*)$/g, '')
                        .replace(/\s*==\s*/g, '==')
                        .split('==')
                    : parse[3]
                        .replace(/(^\s*)|(\s*)$/g, '')
                        .replace(/\s*=\s*/g, '=')
                        .split('=')
                  : [];
                var eval_parse = parse[4] || '';
                var set_data = '';
                var listItem;
                var oneItem = op.get(json, parse[0]);
                if (parse[0].indexOf('.0') + 1) {
                  var items = parse[0].split('.0');
                  var joinItem = items.pop().replace(/^\./, '');
                  var arrayItem = items.join('.0');
                  listItem = op.get(json, arrayItem);
                } else if (parse[0].indexOf('"') + 1) {
                  set_data = parse[0].toString().replace(/"/g, '');
                } else {
                  listItem = op.get(json, parse[0]);
                }
                if (
                  oneItem &&
                  eval_parse &&
                  eval_parse.indexOf('_OBJECT_') + 1
                ) {
                  listItem = eval(
                    eval_parse.replace(
                      /_OBJECT_/g,
                      typeof oneItem === 'object'
                        ? JSON.stringify(oneItem)
                        : typeof oneItem === 'boolean'
                        ? oneItem.toString()
                        : oneItem
                    )
                  );
                }
                if (
                  listItem &&
                  typeof listItem === 'object' &&
                  Array.isArray(listItem) &&
                  listItem.length
                ) {
                  listItem = listItem
                    .map(function(item) {
                      if (item && typeof item === 'string') {
                        return item
                          .replace(/\s+/g, ' ')
                          .replace(/(^\s*)|(\s*)$/g, '');
                      }
                      return item;
                    })
                    .filter(Boolean);
                  if (joinItem) {
                    set_data = listItem
                      .map(function(item) {
                        var set_info = false;
                        if (sup_parse[0] && sup_parse[1]) {
                          if (sup_double) {
                            if (
                              op.get(item, sup_parse[0]) &&
                              op
                                .get(item, sup_parse[0])
                                .toString()
                                .toLowerCase() ===
                                sup_parse[1].toString().toLowerCase()
                            ) {
                              set_info = true;
                            }
                          } else {
                            if (
                              op.get(item, sup_parse[0]) &&
                              op
                                .get(item, sup_parse[0])
                                .toString()
                                .toLowerCase()
                                .indexOf(
                                  sup_parse[1].toString().toLowerCase()
                                ) + 1
                            ) {
                              set_info = true;
                            }
                          }
                        } else {
                          set_info = true;
                        }
                        return set_info ? op.get(item, joinItem) : set_info;
                      })
                      .filter(Boolean)
                      .slice(
                        0,
                        (parse[2] && parseInt(parse[2])) || listItem.length
                      )
                      .join(',');
                  } else {
                    set_data = listItem
                      .slice(
                        0,
                        (parse[2] && parseInt(parse[2])) || listItem.length
                      )
                      .join(',');
                  }
                } else if (listItem) {
                  set_data = listItem.toString();
                }
                if (
                  set_data &&
                  eval_parse &&
                  eval_parse.indexOf('_VALUE_') + 1
                ) {
                  set_data = eval(eval_parse.replace(/_VALUE_/g, set_data));
                }
                op.set(movie, parse[1], set_data);
              });
              if (movie['custom'] && typeof movie['custom'] === 'object') {
                movie['custom']['unique'] =
                  movie['custom']['unique'] === true ||
                  movie['custom']['unique'] === 'true';
                if (movie['custom']['imdb_id']) {
                  movie['custom']['imdb_id'] = movie['custom'][
                    'imdb_id'
                  ].replace(/[^0-9]/g, '');
                }
                [
                  'imdb_id',
                  'tmdb_id',
                  'douban_id',
                  'tvmaze_id',
                  'wa_id',
                  'movie_id'
                ].forEach(function(id) {
                  if (
                    typeof movie['custom'][id] !== 'undefined' &&
                    !movie['custom'][id]
                  ) {
                    delete movie['custom'][id];
                  }
                });
              } else {
                movie['custom'] = { unique: false };
              }
              if (
                movie['title_ru'] &&
                movie['title_en'] &&
                movie['title_ru'] === movie['title_en'] &&
                movie['title_ru'] !== movie['title_ru'].replace(/[^0-9]/g, '')
              ) {
                if (movie['title_en'].replace(/[^а-яё]/gi, '')) {
                  movie['title_en'] = '';
                } else {
                  movie['title_ru'] = '';
                }
              }
              if (movie['description']) {
                movie['description'] = (movie['description'] + '')
                  .replace(/\n\r/g, ' ')
                  .replace(/\n/g, ' ')
                  .replace(/\r/g, ' ')
                  .replace(/\s+/g, ' ')
                  .replace(/(^\s*)|(\s*)$/g, '');
              }
              if (movie['genre']) {
                movie['genre'] = (movie['genre'] + '').replace(' & ', ',');
              }
              if (config.language === 'ru') {
                if (movie['genre']) {
                  movie['genre'] = movie['genre'].toLowerCase();
                  movie['genre'] = movie['genre'].replace('нф ', 'фантастика');
                }
              }
              if (movie['country']) {
                movie['country'] = movie['country']
                  .split(',')
                  .map(function(country) {
                    country = (country + '').trim();
                    if (
                      country &&
                      (country.length === 2 || country.length === 3) &&
                      countries.isValid(country)
                    ) {
                      return countries.getName(country, config.language);
                    }
                    return country;
                  })
                  .filter(Boolean)
                  .join(',');
              }
              if (movie['actor']) {
                movie['actor'] = (movie['actor'] + '')
                  .replace(/\s*,\s*/g, ',')
                  .replace(/\s+/g, ' ')
                  .replace(/(^\s*)|(\s*)$/g, '');
              }
              if (movie['director']) {
                movie['director'] = (movie['director'] + '')
                  .replace(/\s*,\s*/g, ',')
                  .replace(/\s+/g, ' ')
                  .replace(/(^\s*)|(\s*)$/g, '');
              }
              if (!movie['poster']) {
                movie['poster'] = '1';
              } else {
                if (
                  movie['poster'] &&
                  movie['poster'].toLowerCase().indexOf('/st.kp.') + 1
                ) {
                  movie['poster'] = '1';
                } else if (
                  /^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i.test(
                    movie['poster']
                  )
                ) {
                  movie['poster'] = movie['poster'].replace(
                    /^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i,
                    '/$1-$2-$3'
                  );
                } else if (
                  /^.*?media-amazon\.com\/images\/[a-z0-9]\/([a-z0-9@.,_\-]*)$/i.test(
                    movie['poster']
                  )
                ) {
                  movie['poster'] = movie['poster'].replace(
                    /^.*?media-amazon\.com\/images\/[a-z0-9]\/([a-z0-9@.,_\-]*)$/i,
                    '/$1'
                  );
                } else if (
                  /^.*?static\.tvmaze\.com\/uploads\/images\/[a-z_]*\/([0-9]*)\/([0-9]*)\.([a-z0-9]*)$/i.test(
                    movie['poster']
                  )
                ) {
                  movie['poster'] = movie['poster'].replace(
                    /^.*?static\.tvmaze\.com\/uploads\/images\/[a-z_]*\/([0-9]*)\/([0-9]*)\.([a-z0-9]*)$/i,
                    '/$1-$2.$3'
                  );
                } else if (/^\/[a-z0-9]*\.(jpg|png)$/i.test(movie['poster'])) {
                } else {
                  movie['poster'] = '1';
                }
              }
              if (movie['pictures']) {
                if (
                  /^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i.test(
                    movie['pictures']
                  )
                ) {
                  movie['pictures'] = movie['pictures'].replace(
                    /^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i,
                    '/$1-$2-$3'
                  );
                }
              }
              if (movie['rating']) {
                movie['rating'] = ('' + movie['rating']).replace(/,/g, '');
                movie['rating'] =
                  parseFloat(movie['rating']) &&
                  parseFloat(movie['rating']) < 10
                    ? parseInt(parseFloat(movie['rating']) * 10 + '')
                    : parseInt(movie['rating']);
              }
              if (movie['kp_rating']) {
                movie['kp_rating'] = ('' + movie['kp_rating']).replace(
                  /,/g,
                  ''
                );
                movie['kp_rating'] =
                  parseFloat(movie['kp_rating']) &&
                  parseFloat(movie['kp_rating']) < 10
                    ? parseInt(parseFloat(movie['kp_rating']) * 10 + '')
                    : parseInt(movie['kp_rating']);
              }
              if (movie['imdb_rating']) {
                movie['imdb_rating'] = ('' + movie['imdb_rating']).replace(
                  /,/g,
                  ''
                );
                movie['imdb_rating'] =
                  parseFloat(movie['imdb_rating']) &&
                  parseFloat(movie['imdb_rating']) < 10
                    ? parseInt(parseFloat(movie['imdb_rating']) * 10 + '')
                    : parseInt(movie['imdb_rating']);
              }
              if (movie['vote']) {
                movie['vote'] = ('' + movie['vote']).replace(/,/g, '');
                movie['vote'] = parseInt(movie['vote'] + '');
              }
              if (movie['kp_vote']) {
                movie['kp_vote'] = ('' + movie['kp_vote']).replace(/,/g, '');
                movie['kp_vote'] = parseInt(movie['kp_vote'] + '');
              }
              if (movie['imdb_vote']) {
                movie['imdb_vote'] = ('' + movie['imdb_vote']).replace(
                  /,/g,
                  ''
                );
                movie['imdb_vote'] = parseInt(movie['imdb_vote'] + '');
              }
              if (movie['premiere']) {
                if (/[0-9]{1,2}\s+[а-я]+\s+[0-9]{4}/i.test(movie['premiere'])) {
                  var p = movie['premiere'].replace(
                    /([0-9]{1,2})\s+([а-я]+)\s+([0-9]{4})/i,
                    '$3-$2-$1'
                  );
                  [
                    'января',
                    'февраля',
                    'марта',
                    'апреля',
                    'мая',
                    'июня',
                    'июля',
                    'августа',
                    'сентября',
                    'октября',
                    'ноября',
                    'декабря'
                  ].forEach(function(month, i) {
                    if (p.toLowerCase().indexOf(month) + 1) {
                      p = p.toLowerCase().replace(month, i + 1 + '');
                    }
                  });
                  p = p
                    .split('-')
                    .map(function(num) {
                      if (num && parseInt(num) < 10) {
                        num = '0' + num;
                      }
                      return num;
                    })
                    .join('-');
                  movie['premiere'] = p;
                }
                if (/[0-9]{1,2}\s+[a-z]+\s+[0-9]{4}/i.test(movie['premiere'])) {
                  var pp = movie['premiere'].replace(
                    /([0-9]{1,2})\s+([a-z]+)\s+([0-9]{4})/i,
                    '$3-$2-$1'
                  );
                  [
                    'JAN',
                    'FEB',
                    'MAR',
                    'APR',
                    'MAY',
                    'JUN',
                    'JUL',
                    'AUG',
                    'SEP',
                    'OCT',
                    'NOV',
                    'DEC'
                  ].forEach(function(month, i) {
                    if (pp.toUpperCase().indexOf(month) + 1) {
                      p = p.toUpperCase().replace(month, i + 1 + '');
                    }
                  });
                  p = p
                    .split('-')
                    .map(function(num) {
                      if (num && parseInt(num) < 10) {
                        num = '0' + num;
                      }
                      return num;
                    })
                    .join('-');
                  movie['premiere'] = p;
                }
                if (/[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}/i.test(movie['premiere'])) {
                  movie['premiere'] = movie['premiere'] + 'T00:00:00.000Z';
                }
                var year = new Date(movie['premiere']).getFullYear();
                if (!movie['year']) {
                  movie['year'] = !isNaN(year) ? year + '' : '0';
                }
                movie['premiere'] = !isNaN(year)
                  ? Math.floor(
                      new Date(movie['premiere']).getTime() /
                        1000 /
                        60 /
                        60 /
                        24 +
                        719528
                    ) + ''
                  : '0';
              }
              if (
                movie['type'] &&
                (movie['type'].toLowerCase().indexOf('1') + 1 ||
                  movie['type'].toLowerCase().indexOf('tv') + 1 ||
                  movie['type'].toLowerCase().indexOf('show') + 1 ||
                  movie['type'].toLowerCase().indexOf('ser') + 1 ||
                  movie['type'].toLowerCase().indexOf('script') + 1)
              ) {
                movie['type'] = 1;
              } else {
                movie['type'] = 0;
              }
              var req_id =
                (movie['kp_id'] && parseInt(movie['kp_id'])) ||
                (movie['custom'] &&
                  !movie['type'] &&
                  movie['custom']['tmdb_id'] &&
                  parseInt(movie['custom']['tmdb_id']) &&
                  parseInt(movie['custom']['tmdb_id']) + 200000000) ||
                (movie['custom'] &&
                  movie['type'] &&
                  movie['custom']['tmdb_id'] &&
                  parseInt(movie['custom']['tmdb_id']) &&
                  parseInt(movie['custom']['tmdb_id']) + 300000000) ||
                (movie['custom'] &&
                  movie['custom']['imdb_id'] &&
                  parseInt(movie['custom']['imdb_id']) &&
                  parseInt(movie['custom']['imdb_id']) + 400000000) ||
                (movie['custom'] &&
                  movie['custom']['douban_id'] &&
                  parseInt(movie['custom']['douban_id']) &&
                  parseInt(movie['custom']['douban_id']) + 600000000) ||
                (movie['custom'] &&
                  movie['custom']['wa_id'] &&
                  parseInt(movie['custom']['wa_id']) &&
                  parseInt(movie['custom']['wa_id']) + 700000000) ||
                (movie['custom'] &&
                  movie['custom']['tvmaze_id'] &&
                  parseInt(movie['custom']['tvmaze_id']) &&
                  parseInt(movie['custom']['tvmaze_id']) + 800000000) ||
                (movie['custom'] &&
                  movie['custom']['movie_id'] &&
                  parseInt(movie['custom']['movie_id']) &&
                  parseInt(movie['custom']['movie_id']) + 900000000);
              if (
                !req_id ||
                (req_id &&
                  config.movies.skip &&
                  config.movies.skip.length &&
                  config.movies.skip.indexOf(req_id + '') !== -1)
              ) {
                return callback();
              }
              var queries = [];
              if (movie['kp_id'] && parseInt(movie['kp_id'])) {
                queries.push({ id: parseInt(movie['kp_id']) + '' });
              }
              if (
                movie['custom'] &&
                movie['custom']['tmdb_id'] &&
                parseInt(movie['custom']['tmdb_id'])
              ) {
                queries.push({
                  type: movie['type'],
                  id: 'custom.tmdb_id',
                  'custom.tmdb_id':
                    movie['custom']['tmdb_id'].replace(/[^0-9]/g, '') + ''
                });
              }
              if (
                movie['custom'] &&
                movie['custom']['imdb_id'] &&
                parseInt(movie['custom']['imdb_id'])
              ) {
                queries.push({
                  id: 'custom.imdb_id',
                  'custom.imdb_id':
                    movie['custom']['imdb_id'].replace(/[^0-9]/g, '') + ''
                });
              }
              if (
                movie['custom'] &&
                movie['custom']['douban_id'] &&
                parseInt(movie['custom']['douban_id'])
              ) {
                queries.push({
                  id: 'custom.douban_id',
                  'custom.douban_id':
                    movie['custom']['douban_id'].replace(/[^0-9]/g, '') + ''
                });
              }
              if (
                movie['custom'] &&
                movie['custom']['tvmaze_id'] &&
                parseInt(movie['custom']['tvmaze_id'])
              ) {
                queries.push({
                  id: 'custom.tvmaze_id',
                  'custom.tvmaze_id':
                    movie['custom']['tvmaze_id'].replace(/[^0-9]/g, '') + ''
                });
              }
              if (
                movie['custom'] &&
                movie['custom']['wa_id'] &&
                parseInt(movie['custom']['wa_id'])
              ) {
                queries.push({
                  id: 'custom.wa_id',
                  'custom.wa_id':
                    movie['custom']['wa_id'].replace(/[^0-9]/g, '') + ''
                });
              }
              if (
                movie['custom'] &&
                movie['custom']['movie_id'] &&
                parseInt(movie['custom']['movie_id'])
              ) {
                queries.push({
                  id: 'custom.movie_id',
                  'custom.movie_id':
                    movie['custom']['movie_id'].replace(/[^0-9]/g, '') + ''
                });
              }
              var current_movie = null;
              async.eachOfLimit(
                queries,
                1,
                function(query, query_index, callback) {
                  if (current_movie) {
                    return callback('STOP');
                  }
                  var req = {};
                  req['certainly'] = true;
                  var w = '';
                  if (query.id.indexOf('custom.') + 1 && query[query.id]) {
                    w = query.id + " = '" + query[query.id] + "'";
                    w += query.type
                      ? ' AND type = ' +
                        (query.type.toString() === '1' ? '1' : '0')
                      : '';
                  } else {
                    w = '`kp_id` = ' + query.id + '';
                  }
                  pool.getConnection(function(err, connection) {
                    if (err) {
                      connection.release();
                      return callback();
                    }
                    connection.query(
                      'SELECT * FROM rt_' +
                        config.domain.replace(/[^a-z0-9]/g, '_') +
                        ' WHERE ' +
                        w +
                        ' LIMIT 0,1 OPTION max_matches = 1',
                      function(err, rt) {
                        connection.release();
                        if (err) {
                          console.error(err);
                          return callback('STOP');
                        }
                        if (rt && rt.length) {
                          current_movie = Object.assign({}, rt[0]);
                          return callback('STOP');
                        }
                        return callback();
                      }
                    );
                  });
                },
                function() {
                  if (current_movie) {
                    delete current_movie['all_movies'];
                    delete movie['all_movies'];
                    var cm = Object.assign({}, current_movie);
                    delete cm['custom'];
                    var commit_movie = JSON.stringify(
                      Object.keys(cm)
                        .sort()
                        .reduce(function(obj, key) {
                          obj[key] = cm[key];
                          return obj;
                        }, {})
                    );
                    var parse_movie = Object.assign({}, movie);
                    var custom_no_change = false;
                    Object.keys(parse_movie).forEach(function(k) {
                      if (!parse_movie[k]) {
                        delete parse_movie[k];
                      }
                    });
                    Object.keys(current_movie).forEach(function(k) {
                      if (!current_movie[k] && parse_movie[k]) {
                        delete current_movie[k];
                      }
                    });
                    if (
                      (current_movie['poster'] === '1' ||
                        current_movie['poster'] === 1 ||
                        current_movie['poster'] === '0' ||
                        current_movie['poster'] === 0) &&
                      parse_movie['poster']
                    ) {
                      delete current_movie['poster'];
                    }
                    if (
                      current_movie['quality'] &&
                      parse_movie['quality'] &&
                      current_movie['quality'] !== parse_movie['quality']
                    ) {
                      delete current_movie['quality'];
                    }
                    if (
                      current_movie['translate'] &&
                      parse_movie['translate'] &&
                      current_movie['translate'] !== parse_movie['translate']
                    ) {
                      delete current_movie['translate'];
                    }
                    var current_movie_custom = {};
                    if (current_movie.custom) {
                      if (
                        current_movie.custom &&
                        typeof current_movie.custom === 'string'
                      ) {
                        current_movie_custom = JSON.parse(current_movie.custom);
                      } else {
                        current_movie_custom = Object.assign(
                          {},
                          current_movie.custom
                        );
                      }
                    }
                    if (current_movie.custom && parse_movie.custom) {
                      var parse_movie_custom = {};
                      if (
                        parse_movie.custom &&
                        typeof parse_movie.custom === 'string'
                      ) {
                        parse_movie_custom = JSON.parse(parse_movie.custom);
                      } else {
                        parse_movie_custom = Object.assign(
                          {},
                          parse_movie.custom
                        );
                      }
                      [1, 2, 3, 4, 5].forEach(function(i) {
                        if (
                          current_movie_custom['player' + i] &&
                          parse_movie_custom['player' + i]
                        ) {
                          delete current_movie_custom['player' + i];
                          if (parse_movie_custom['player' + i] === 'none') {
                            delete parse_movie_custom['player' + i];
                          }
                        }
                      });
                      [
                        'imdb_id',
                        'tmdb_id',
                        'douban_id',
                        'tvmaze_id',
                        'wa_id',
                        'movie_id'
                      ].forEach(function(id) {
                        if (
                          typeof parse_movie_custom[id] !== 'undefined' &&
                          !parse_movie_custom[id]
                        ) {
                          delete parse_movie_custom[id];
                        }
                        if (
                          typeof current_movie_custom[id] !== 'undefined' &&
                          !current_movie_custom[id]
                        ) {
                          delete current_movie_custom[id];
                        }
                      });
                      if (typeof parse_movie_custom['unique'] !== 'undefined') {
                        parse_movie_custom['unique'] =
                          parse_movie_custom['unique'] === true ||
                          parse_movie_custom['unique'] === 'true';
                      }
                      if (
                        typeof current_movie_custom['unique'] !== 'undefined'
                      ) {
                        current_movie_custom['unique'] =
                          current_movie_custom['unique'] === true ||
                          current_movie_custom['unique'] === 'true';
                      }
                      if (
                        typeof parse_movie_custom['unique'] !== 'undefined' &&
                        typeof current_movie_custom['unique'] !== 'undefined' &&
                        parse_movie_custom['unique'] !==
                          current_movie_custom['unique']
                      ) {
                        delete current_movie_custom['unique'];
                      }
                      current_movie.custom = Object.assign(
                        {},
                        parse_movie_custom,
                        current_movie_custom
                      );
                      if (
                        JSON.stringify(
                          Object.keys(current_movie.custom)
                            .sort()
                            .reduce(function(obj, key) {
                              obj[key] = current_movie.custom[key];
                              return obj;
                            }, {})
                        ) ===
                        JSON.stringify(
                          Object.keys(current_movie_custom)
                            .sort()
                            .reduce(function(obj, key) {
                              obj[key] = current_movie_custom[key];
                              return obj;
                            }, {})
                        )
                      ) {
                        custom_no_change = true;
                      }
                    }
                    if (
                      current_movie_custom &&
                      typeof current_movie_custom.unique === 'boolean' &&
                      current_movie_custom.unique === false
                    ) {
                      if (
                        current_movie['title_ru'] &&
                        parse_movie['title_ru'] &&
                        current_movie['title_ru'] !== parse_movie['title_ru']
                      ) {
                        delete current_movie['title_ru'];
                      }
                      if (
                        current_movie['title_en'] &&
                        parse_movie['title_en'] &&
                        current_movie['title_en'] !== parse_movie['title_en']
                      ) {
                        delete current_movie['title_en'];
                      }
                      if (
                        current_movie['description'] &&
                        parse_movie['description'] &&
                        current_movie['description'] !==
                          parse_movie['description'] &&
                        current_movie['description'] <
                          parse_movie['description']
                      ) {
                        delete current_movie['description'];
                      }
                    }
                    var update_movie = Object.assign(
                      {},
                      parse_movie,
                      current_movie
                    );
                    [
                      'year',
                      'rating',
                      'vote',
                      'kp_rating',
                      'kp_vote',
                      'imdb_rating',
                      'imdb_vote',
                      'premiere'
                    ].forEach(function(attr_uint) {
                      var current =
                        (typeof current_movie[attr_uint] !== 'undefined' &&
                          current_movie[attr_uint] &&
                          parseFloat(current_movie[attr_uint])) ||
                        0;
                      var parse =
                        (typeof parse_movie[attr_uint] !== 'undefined' &&
                          parse_movie[attr_uint] &&
                          parseFloat(parse_movie[attr_uint])) ||
                        0;
                      if (
                        attr_uint === 'rating' ||
                        attr_uint === 'kp_rating' ||
                        attr_uint === 'imdb_rating'
                      ) {
                        if (parse && parse > 100) {
                          parse = 0;
                        }
                        if (!parse) {
                          return;
                        }
                        if (parse < 10) {
                          parse = parseInt(parse * 10 + '');
                        }
                        update_movie[attr_uint] = parse;
                        return;
                      }
                      if (
                        attr_uint === 'year' ||
                        attr_uint === 'premiere' ||
                        attr_uint === 'vote' ||
                        attr_uint === 'kp_vote' ||
                        attr_uint === 'imdb_vote'
                      ) {
                        parse = parseInt(parse + '');
                      }
                      if (parse > current) {
                        update_movie[attr_uint] = parseInt(parse);
                      }
                      if (
                        attr_uint === 'year' &&
                        typeof update_movie['year'] !== 'undefined'
                      ) {
                        update_movie['year'] = update_movie['year'] + '';
                      }
                    });
                    ['country', 'director', 'genre', 'actor'].forEach(function(
                      attr_string
                    ) {
                      var parse =
                        (typeof parse_movie[attr_string] !== 'undefined' &&
                          parse_movie[attr_string]) ||
                        '';
                      var current = (
                        (typeof current_movie[attr_string] !== 'undefined' &&
                          current_movie[attr_string]) ||
                        ''
                      ).replace('_empty', '');
                      if (parse.length > current.length) {
                        update_movie[attr_string] = parse_movie[attr_string];
                      }
                    });
                    var cm2 = Object.assign({}, update_movie);
                    delete cm2['custom'];
                    var commit_movie2 = JSON.stringify(
                      Object.keys(cm2)
                        .sort()
                        .reduce(function(obj, key) {
                          obj[key] = cm2[key];
                          return obj;
                        }, {})
                    );
                    if (custom_no_change && commit_movie === commit_movie2) {
                      console.log(
                        '[REALTIME]',
                        id_index + 1,
                        '/',
                        ids.length,
                        iter_req ? '[' + iter_req + ']' : '',
                        'NO UPDATE',
                        current_movie.id,
                        'PROXY',
                        movies_proxy || false,
                        movies_captcha || ''
                      );
                      return callback();
                    }
                    console.log(
                      '[REALTIME]',
                      id_index + 1,
                      '/',
                      ids.length,
                      iter_req ? '[' + iter_req + ']' : '',
                      'UPDATE MOVIE',
                      current_movie.id,
                      'PROXY',
                      movies_proxy || false
                    );
                    console.log(update_movie);
                    CP_save.save(update_movie, 'rt', function(err, result) {
                      update_m++;
                      if (err) console.log(err);
                      return callback(err);
                    });
                  } else {
                    if (!movie['title_ru'] && !movie['title_en']) {
                      console.log(
                        '[REALTIME]',
                        id_index + 1,
                        '/',
                        ids.length,
                        iter_req ? '[' + iter_req + ']' : '',
                        'NO SAVE',
                        req_id,
                        'PROXY',
                        movies_proxy || false
                      );
                      return callback();
                    }
                    console.log(
                      '[REALTIME]',
                      id_index + 1,
                      '/',
                      ids.length,
                      iter_req ? '[' + iter_req + ']' : '',
                      'SAVE MOVIE',
                      req_id,
                      'PROXY',
                      movies_proxy || false
                    );
                    console.log(movie);
                    CP_save.save(movie, 'rt', function(err, result) {
                      added_m++;
                      if (err) console.log(err);
                      return callback(err);
                    });
                  }
                }
              );
            }
          },
          function() {
            console.log('[REALTIME]', 'ADDED:', added_m);
            console.log('[REALTIME]', 'UPDATE:', update_m);
            return callback();
          }
        );
      }
    },
    function() {
      process.env['NO_CACHE'] = undefined;
      console.log(
        '[REALTIME]',
        new Date()
          .toJSON()
          .replace('T', ' ')
          .split('.')[0],
        '[',
        Math.ceil(os.freemem()),
        'MB ]',
        os.loadavg(1).toFixed(2),
        os.loadavg(5).toFixed(2),
        os.loadavg(15).toFixed(2)
      );
      console.timeEnd('[REALTIME] DONE');
      return process.exit(0);
    }
  );
})();

/**
 * Valid JSON.
 *
 */

function tryParseJSON(jsonString) {
  try {
    if (jsonString && typeof jsonString === 'string') {
      if (jsonString.indexOf('<?xml') + 1) {
        var result = convert.xml2js(jsonString, { compact: true });
        if (result && typeof result === 'object') {
          return result;
        }
      }
      var o = JSON.parse(jsonString);
      if (o && typeof o === 'object') {
        return o;
      }
    } else {
      if (jsonString && typeof jsonString === 'object') {
        return jsonString;
      }
    }
  } catch (e) {}
  return null;
}