LoweBowe/CinemaPress
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity
CinemaPress/lib/CP_movies.js

1608 lines
59 KiB
JavaScript
Raw Normal View History

Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
'use strict';
/**
* Configuration dependencies.
*/
var config = require('../config/production/config');
Object.keys(config).length === 0 &&
(config = require('../config/production/config.backup'));
var modules = require('../config/production/modules');
Object.keys(modules).length === 0 &&
(modules = require('../config/production/modules.backup'));
/**
* Module dependencies.
*/
var CP_get = require('./CP_get');
var CP_save = require('./CP_save');
/**
* Node dependencies.
*/
var fs = require('fs');
var path = require('path');
var os = require('os-utils');
var op = require('object-path');
var async = require('async');
var adop = require('adop');
var axios = require('axios');
var convert = require('xml-js');
var htmlmetaparser = require('htmlmetaparser');
var countries = require('i18n-iso-countries');
var htmlparser2 = require('htmlparser2');
var LRU = require('lru-cache');
var cache = new LRU({ max: 1000 });
var HttpsProxyAgent = require('https-proxy-agent');
var sphinx = require('sphinx');
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
var axiosCookieJarSupport = require('axios-cookiejar-support').default;
var tough = require('tough-cookie');
var Cookie = tough.Cookie;
axiosCookieJarSupport(axios);
var cookieJar = new tough.CookieJar();
if (config.movies.cookies) {
try {
var cookies = JSON.parse(config.movies.cookies);
Object.keys(cookies).forEach(function(domain) {
cookies[domain].forEach(function(cookie) {
Fixed cookie, fixed type title
2021-02-17 06:39:57 +01:00
if (!cookie) return;
if (cookie.name) {
cookie.key = cookie.name;
}
if (cookie.expirationDate) {
cookie.expires = new Date(parseInt(cookie.expirationDate) * 1000);
}
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
cookieJar.setCookie(Cookie.fromJSON(cookie), domain, function(e, c) {});
});
});
} catch (e) {
cookieJar = undefined;
console.error(e);
}
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
/**
* Global env.
*/
try {
var p = tryParseJSON(
fs.readFileSync(
path.join(path.dirname(__filename), '..', 'process.json'),
'utf8'
)
);
var e = p.apps[0].env;
for (var prop in e) {
if (e.hasOwnProperty(prop)) {
process.env[prop] = e[prop];
}
}
} catch (err) {
return console.log('NOT FILE PROCESS DATA');
}
process.env['NO_CACHE'] = true;
var run = process.argv && typeof process.argv[2] !== 'undefined';
var timeZone = new Date();
var hour = new Date(timeZone).getHours() + 1;
Added stop receiving info
2021-02-15 08:25:53 +01:00
console.time('[REALTIME] DONE');
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
console.log(
Added stop receiving info
2021-02-15 08:25:53 +01:00
'[REALTIME]',
new Date()
.toJSON()
.replace('T', ' ')
.split('.')[0],
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
'[',
Math.ceil(os.freemem()),
'MB ]',
os.loadavg(1).toFixed(2),
os.loadavg(5).toFixed(2),
os.loadavg(15).toFixed(2)
);
if (Math.ceil(os.freemem()) < 50) {
console.log('The server is overloaded to start get movies.');
return process.exit(0);
}
/**
* Get movie information.
*
*/
(function getInfo() {
if (!config.movies.cron) return process.exit();
var tasks = [];
config.movies.cron.forEach(function(task) {
var parse = task
.replace(/(^\s*)|(\s*)$/g, '')
.replace(/\s*~\s*/g, '~')
.split('~');
if (task.charAt(0) === '#') return;
tasks.push({
hour: parse[0] && parseInt(parse[0]) ? parseInt(parse[0]) : 0,
page: parse[1] || '',
path: (parse[2] || '').split('<>')[0].trim(),
id: parse[3] || '',
info: parse.slice(4),
type_id: ((parse[2] || '').split('<>')[1] || '').trim(),
ids: []
});
});
var update_m = 0;
var added_m = 0;
async.eachOfLimit(
tasks,
1,
function(task, task_index, callback) {
if (task.hour && task.hour % hour) {
return callback();
}
if (task.hour === 0 && !run) {
return callback();
}
var i = 1;
var fail_req = 5;
var next_req = 1;
var prev_req = 0;
var some_req = 0;
var retry_req = '';
var iter_req = 0;
var ids = cache.get(task.page + task.path + task.type_id) || [];
if (ids && ids.length) {
fail_req = 0;
next_req = 0;
}
async.forever(
function(next) {
if (
task.page === '' ||
task.page === 'realtime' ||
task.page === 'rt' ||
task.page === 'database' ||
task.page === 'db'
) {
CP_get.movies(
{ certainly: true },
100,
'kinopoisk-id-up',
i,
false,
function(err, rt) {
if (ids.length && !(ids.length % 1000)) {
console.log('[REALTIME]', ids.length);
}
if (err) {
console.error(err);
return next('STOP');
}
if (rt && rt.length) {
rt.forEach(function(r) {
var id = '';
if (task.path.indexOf('custom.') + 1) {
var cst = {};
try {
cst = r.custom ? JSON.parse(r.custom) : {};
} catch (e) {}
id = op.get({ custom: cst }, task.path);
if (id) {
task.type_id = task.path;
}
} else {
id = r['kp_id'];
task.type_id = 'kp_id';
}
if (id && ids.indexOf(id + '') === -1) {
ids.push(id + '');
}
});
iter_req = iter_req + ids.length;
ids_iteration(function() {
some_req = 0;
ids = [];
i++;
return next();
});
} else {
return next('STOP');
}
}
);
} else {
if (task.page.indexOf('[page]') === -1 && i > 1) {
console.log('[PAGE] 1');
return next('STOP');
}
if (fail_req <= 0) {
if (next_req) {
i++;
fail_req = 5;
next_req = 0;
retry_req = '';
return next();
}
console.log('[FAIL] REQUEST');
return next('STOP');
}
if (some_req >= 5) {
console.log('[SOME] LENGTH');
return next('STOP');
}
if (prev_req === ids.length) {
some_req++;
} else {
retry_req = '';
some_req = 0;
prev_req = 0;
}
prev_req = ids.length;
var url_req = task.page.replace('[page]', i);
if (retry_req) {
Added clear cache to url
2021-02-21 08:58:42 +01:00
url_req =
retry_req +
(retry_req.indexOf('?') + 1 ? '&' : '?') +
'clear-cache-request=' +
Math.random()
.toString(36)
.slice(2);
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
} else {
console.log('[REALTIME]', ids.length, url_req);
}
var opt = {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0'
}
};
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
if (config.movies.cookies && cookieJar) {
opt.jar = cookieJar;
Fixed cookie, fixed type title
2021-02-17 06:39:57 +01:00
opt.withCredentials = true;
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
var hostname = require('url').parse(url_req).hostname;
Added stop receiving info
2021-02-15 08:25:53 +01:00
var movies_proxy = '';
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
if (config.movies.proxy && config.movies.proxy.length) {
var proxies = config.movies.proxy.filter(function(p) {
var only_proxy = p.split('<>')[0].trim();
var only_domain = (p.split('<>')[1] || '').trim();
if (only_domain) {
if (only_proxy && only_domain === hostname) {
return true;
}
} else if (only_proxy) {
return true;
}
return false;
});
var proxy =
proxies && proxies.length
? proxies[Math.floor(Math.random() * proxies.length)]
: '';
if (proxy) {
var proxy_obj = {};
if (proxy.indexOf('@') + 1) {
var host_auth = proxy.split('@');
proxy_obj['host'] = host_auth[1].split(':')[0];
proxy_obj['port'] = parseInt(host_auth[1].split(':')[1]);
proxy_obj['auth'] = host_auth[0];
} else {
proxy_obj['host'] = proxy.split(':')[0];
proxy_obj['port'] = parseInt(proxy.split(':')[1]);
}
opt.httpsAgent = new HttpsProxyAgent(proxy_obj);
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
movies_proxy =
proxy.indexOf('@') + 1 ? proxy.split('@')[1] : proxy;
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
}
}
axios(url_req, opt)
.then(function(r) {
if (!r || !r.data) {
if (retry_req) {
i++;
retry_req = '';
} else {
retry_req = url_req;
}
console.error(
'[REALTIME]',
'STOP PAGE (' + (6 - fail_req) + ' fails)',
Added stop receiving info
2021-02-15 08:25:53 +01:00
url_req,
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
'PROXY',
movies_proxy || false
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
fail_req = fail_req - 1;
setTimeout(function() {
return next();
}, 5000);
} else if (r.data) {
if (
typeof r.data === 'string' &&
r.data.indexOf('<html') + 1
) {
if (task.path) {
var ids_list = r.data.match(task.path.split('<>')[0]);
if (ids_list && ids_list.length) {
ids_list.forEach(function(id) {
id = id.trim();
if (id && ids.indexOf(id) === -1) {
ids.push(id);
}
});
}
}
i++;
return next();
} else {
var r_data = tryParseJSON(r.data);
Added get data from url
2021-03-12 08:50:45 +01:00
if (r_data && (!task.id || task.id === '[url]')) {
if (!task.path || task.id === '[url]') {
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
if (
typeof r_data === 'object' &&
Array.isArray(r_data)
) {
ids = r_data;
iter_req = iter_req + ids.length;
ids_iteration(function() {
some_req = 0;
ids = [];
i++;
return next();
});
} else {
i++;
return next();
}
} else {
var get_array = op.get(r_data, task.path);
if (
typeof get_array === 'object' &&
Array.isArray(get_array)
) {
ids = get_array;
iter_req = iter_req + ids.length;
ids_iteration(function() {
some_req = 0;
ids = [];
i++;
return next();
});
} else {
i++;
return next();
}
}
} else {
var all = adop(r_data, [
{
name: 'id',
path: task.path.split('<>')[0],
type: task.path.split('<>')[1] || undefined,
regex: task.path.split('<>')[2] || undefined
}
]);
if (all && all.length) {
all.forEach(function(a) {
if (
a &&
a.id &&
a.id !== 'null' &&
a.id !== 'false' &&
a.id !== 'undefined' &&
a.id !== 'n/a' &&
a.id !== 'N/A' &&
ids.indexOf(a.id) === -1
) {
ids.push(a.id);
}
});
if (!ids.length) {
some_req = some_req + 1;
ids = [];
i++;
return next();
}
iter_req = iter_req + ids.length;
ids_iteration(function() {
some_req = 0;
ids = [];
i++;
return next();
});
} else {
if (retry_req) {
i++;
retry_req = '';
} else {
retry_req = url_req;
}
console.error(
'[REALTIME]',
'STOP PAGE (' + (6 - fail_req) + ' fails)',
Added stop receiving info
2021-02-15 08:25:53 +01:00
url_req,
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
'PROXY',
movies_proxy || false
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
fail_req = fail_req - 1;
setTimeout(function() {
return next();
}, 5000);
}
}
}
} else {
return next();
}
})
.catch(function(err) {
if (retry_req) {
i++;
retry_req = '';
} else {
retry_req = url_req;
}
console.error(
'[REALTIME]',
'STOP PAGE (' + (6 - fail_req) + ' fails)',
url_req,
err.response && err.response.status + '',
Added stop receiving info
2021-02-15 08:25:53 +01:00
err.response && err.response.statusText + '',
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
'PROXY',
movies_proxy || false
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
fail_req = fail_req - 1;
setTimeout(function() {
return next();
}, 5000);
});
}
},
function() {
ids_iteration(function() {
callback();
});
}
);
function ids_iteration(callback) {
var pool = sphinx.createPool({});
cache.set(task.page + task.path + task.type_id, ids);
async.eachOfLimit(
ids,
5,
function(id, id_index, callback) {
if (
task.id &&
!/\[[a-z0-9_]+?]/i.test(task.id + '') &&
id_index >= 1
) {
if (
!(
task.page === '' ||
task.page === 'realtime' ||
task.page === 'rt' ||
task.page === 'database' ||
task.page === 'db'
)
) {
return callback();
}
}
var task_type_id = task.type_id || '';
var task_url =
typeof id === 'object'
? ''
: (task.id + '').replace(/\[[a-z0-9_]+?]/i, id + '');
if (task_url) {
var opt = {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0'
}
};
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
if (config.movies.cookies && cookieJar) {
opt.jar = cookieJar;
Fixed cookie, fixed type title
2021-02-17 06:39:57 +01:00
opt.withCredentials = true;
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
var hostname = require('url').parse(task_url).hostname;
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
var movies_proxy = '';
Added detect captcha
2021-02-16 09:21:10 +01:00
var movies_captcha = '';
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
if (config.movies.proxy && config.movies.proxy.length) {
var proxies = config.movies.proxy.filter(function(p) {
var only_proxy = p.split('<>')[0].trim();
var only_domain = (p.split('<>')[1] || '').trim();
if (only_domain) {
if (only_proxy && only_domain === hostname) {
return true;
}
} else if (only_proxy) {
return true;
}
return false;
});
var proxy =
proxies && proxies.length
? proxies[Math.floor(Math.random() * proxies.length)]
: '';
if (proxy) {
var proxy_obj = {};
if (proxy.indexOf('@') + 1) {
var host_auth = proxy.split('@');
proxy_obj['host'] = host_auth[1].split(':')[0];
proxy_obj['port'] = parseInt(host_auth[1].split(':')[1]);
proxy_obj['auth'] = host_auth[0];
} else {
proxy_obj['host'] = proxy.split(':')[0];
proxy_obj['port'] = parseInt(proxy.split(':')[1]);
}
opt.httpsAgent = new HttpsProxyAgent(proxy_obj);
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
movies_proxy =
proxy.indexOf('@') + 1 ? proxy.split('@')[1] : proxy;
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
}
}
axios(task_url, opt)
.then(function(j) {
if (!j || !j.data) {
return callback();
}
if (
typeof j.data === 'string' &&
j.data.indexOf('<html') + 1
) {
var handler = new htmlmetaparser.Handler(
function(err, json) {
if (!json || typeof json !== 'object') {
return callback();
}
Added detect captcha
2021-02-16 09:21:10 +01:00
if (
json &&
json.html &&
json.html.title &&
json.html.title === 'Ой!'
) {
movies_captcha = 'CAPTCHA';
json = {};
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
movie_data(json, function() {
return callback();
});
},
{ url: task_url }
);
var parser = new htmlparser2.Parser(handler, {
decodeEntities: true
});
parser.write(j.data);
parser.done();
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
} else {
var json = tryParseJSON(j.data);
if (!json || typeof json !== 'object') {
return callback();
}
Added get data from url
2021-03-12 08:50:45 +01:00
if (task.id === '[url]') {
var movies_on_page = op.get(json, task.path);
if (Array.isArray(movies_on_page)) {
async.eachOfLimit(
movies_on_page,
5,
function(movie, id_index, callback) {
movie_data(movie, function() {
return callback();
});
},
function() {
return callback();
}
);
} else {
return callback();
}
} else {
movie_data(json, function() {
return callback();
});
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
}
})
.catch(function(err) {
console.log(
'[REALTIME]',
id_index + 1,
'/',
ids.length,
iter_req ? '[' + iter_req + ']' : '',
'ERROR MOVIE',
task_url,
err.response && err.response.status + '',
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
err.response && err.response.statusText + '',
'PROXY',
movies_proxy || false
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
return callback();
});
} else if (typeof id === 'object') {
movie_data(id, function() {
return callback();
});
} else {
movie_data({}, function() {
return callback();
});
}
function movie_data(json, callback) {
var movie = {};
if (task_type_id) {
if (task_type_id === 'id' || task_type_id === 'kp_id') {
movie['kp_id'] = id.replace(/[^0-9]/g, '');
} else if (task_type_id.indexOf('custom.') + 1) {
op.set(movie, task_type_id, id.replace(/[^0-9]/g, ''));
}
}
task.info.forEach(function(info) {
var parse = info
.replace(/(^\s*)|(\s*)$/g, '')
.replace(/\s*<>\s*/g, '<>')
.split('<>');
var sup_double = !!(parse[3] && parse[3].indexOf('==') + 1);
var sup_parse = parse[3]
? parse[3].indexOf('==') + 1
? parse[3]
.replace(/(^\s*)|(\s*)$/g, '')
.replace(/\s*==\s*/g, '==')
.split('==')
: parse[3]
.replace(/(^\s*)|(\s*)$/g, '')
.replace(/\s*=\s*/g, '=')
.split('=')
: [];
var eval_parse = parse[4] || '';
var set_data = '';
var listItem;
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
var oneItem = op.get(json, parse[0]);
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
if (parse[0].indexOf('.0') + 1) {
var items = parse[0].split('.0');
var joinItem = items.pop().replace(/^\./, '');
var arrayItem = items.join('.0');
listItem = op.get(json, arrayItem);
} else if (parse[0].indexOf('"') + 1) {
set_data = parse[0].toString().replace(/"/g, '');
} else {
listItem = op.get(json, parse[0]);
}
if (
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
oneItem &&
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
eval_parse &&
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
eval_parse.indexOf('_OBJECT_') + 1
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
) {
listItem = eval(
eval_parse.replace(
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
/_OBJECT_/g,
typeof oneItem === 'object'
? JSON.stringify(oneItem)
: typeof oneItem === 'boolean'
? oneItem.toString()
: oneItem
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
)
);
}
if (
listItem &&
typeof listItem === 'object' &&
Array.isArray(listItem) &&
listItem.length
) {
listItem = listItem
.map(function(item) {
if (item && typeof item === 'string') {
return item
.replace(/\s+/g, ' ')
.replace(/(^\s*)|(\s*)$/g, '');
}
return item;
})
.filter(Boolean);
if (joinItem) {
set_data = listItem
.map(function(item) {
var set_info = false;
if (sup_parse[0] && sup_parse[1]) {
if (sup_double) {
if (
op.get(item, sup_parse[0]) &&
op
.get(item, sup_parse[0])
.toString()
.toLowerCase() ===
sup_parse[1].toString().toLowerCase()
) {
set_info = true;
}
} else {
if (
op.get(item, sup_parse[0]) &&
op
.get(item, sup_parse[0])
.toString()
.toLowerCase()
.indexOf(
sup_parse[1].toString().toLowerCase()
) + 1
) {
set_info = true;
}
}
} else {
set_info = true;
}
return set_info ? op.get(item, joinItem) : set_info;
})
.filter(Boolean)
.slice(
0,
(parse[2] && parseInt(parse[2])) || listItem.length
)
.join(',');
} else {
set_data = listItem
.slice(
0,
(parse[2] && parseInt(parse[2])) || listItem.length
)
.join(',');
}
} else if (listItem) {
set_data = listItem.toString();
}
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
if (
set_data &&
eval_parse &&
eval_parse.indexOf('_VALUE_') + 1
) {
set_data = eval(eval_parse.replace(/_VALUE_/g, set_data));
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
op.set(movie, parse[1], set_data);
});
if (movie['custom'] && typeof movie['custom'] === 'object') {
movie['custom']['unique'] =
movie['custom']['unique'] === true ||
movie['custom']['unique'] === 'true';
if (movie['custom']['imdb_id']) {
movie['custom']['imdb_id'] = movie['custom'][
'imdb_id'
].replace(/[^0-9]/g, '');
}
[
'imdb_id',
'tmdb_id',
'douban_id',
'tvmaze_id',
'wa_id',
'movie_id'
].forEach(function(id) {
if (
typeof movie['custom'][id] !== 'undefined' &&
!movie['custom'][id]
) {
delete movie['custom'][id];
}
});
} else {
movie['custom'] = { unique: false };
}
if (
movie['title_ru'] &&
movie['title_en'] &&
movie['title_ru'] === movie['title_en'] &&
movie['title_ru'] !== movie['title_ru'].replace(/[^0-9]/g, '')
) {
if (movie['title_en'].replace(/[^а-яё]/gi, '')) {
movie['title_en'] = '';
} else {
movie['title_ru'] = '';
}
}
if (movie['description']) {
movie['description'] = (movie['description'] + '')
.replace(/\n\r/g, ' ')
.replace(/\n/g, ' ')
.replace(/\r/g, ' ')
.replace(/\s+/g, ' ')
.replace(/(^\s*)|(\s*)$/g, '');
}
if (movie['genre']) {
movie['genre'] = (movie['genre'] + '').replace(' & ', ',');
}
if (config.language === 'ru') {
if (movie['genre']) {
movie['genre'] = movie['genre'].toLowerCase();
movie['genre'] = movie['genre'].replace('нф ', 'фантастика');
}
}
if (movie['country']) {
movie['country'] = movie['country']
.split(',')
.map(function(country) {
country = (country + '').trim();
if (
country &&
(country.length === 2 || country.length === 3) &&
countries.isValid(country)
) {
return countries.getName(country, config.language);
}
return country;
})
.filter(Boolean)
.join(',');
}
Changed log
2021-03-13 07:13:24 +01:00
if (movie['actor']) {
movie['actor'] = (movie['actor'] + '')
.replace(/\s*,\s*/g, ',')
.replace(/\s+/g, ' ')
.replace(/(^\s*)|(\s*)$/g, '');
}
if (movie['director']) {
movie['director'] = (movie['director'] + '')
.replace(/\s*,\s*/g, ',')
.replace(/\s+/g, ' ')
.replace(/(^\s*)|(\s*)$/g, '');
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
if (!movie['poster']) {
movie['poster'] = '1';
} else {
if (
movie['poster'] &&
movie['poster'].toLowerCase().indexOf('/st.kp.') + 1
) {
movie['poster'] = '1';
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
} else if (
/^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i.test(
movie['poster']
)
) {
movie['poster'] = movie['poster'].replace(
/^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i,
'/$1-$2-$3'
);
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
} else if (
/^.*?media-amazon\.com\/images\/[a-z0-9]\/([a-z0-9@.,_\-]*)$/i.test(
movie['poster']
)
) {
movie['poster'] = movie['poster'].replace(
/^.*?media-amazon\.com\/images\/[a-z0-9]\/([a-z0-9@.,_\-]*)$/i,
'/$1'
);
} else if (
/^.*?static\.tvmaze\.com\/uploads\/images\/[a-z_]*\/([0-9]*)\/([0-9]*)\.([a-z0-9]*)$/i.test(
movie['poster']
)
) {
movie['poster'] = movie['poster'].replace(
/^.*?static\.tvmaze\.com\/uploads\/images\/[a-z_]*\/([0-9]*)\/([0-9]*)\.([a-z0-9]*)$/i,
'/$1-$2.$3'
);
} else if (/^\/[a-z0-9]*\.(jpg|png)$/i.test(movie['poster'])) {
} else {
movie['poster'] = '1';
}
}
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
if (movie['pictures']) {
if (
/^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i.test(
movie['pictures']
)
) {
movie['pictures'] = movie['pictures'].replace(
/^.*?avatars\.mds\.yandex\.net\/(get-kinopoisk-image|get-kino-vod-films-gallery)\/([0-9]*)\/([a-z0-9\-]*)\/[0-9x_]*$/i,
'/$1-$2-$3'
);
}
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
if (movie['rating']) {
movie['rating'] = ('' + movie['rating']).replace(/,/g, '');
movie['rating'] =
parseFloat(movie['rating']) &&
parseFloat(movie['rating']) < 10
? parseInt(parseFloat(movie['rating']) * 10 + '')
: parseInt(movie['rating']);
}
if (movie['kp_rating']) {
movie['kp_rating'] = ('' + movie['kp_rating']).replace(
/,/g,
''
);
movie['kp_rating'] =
parseFloat(movie['kp_rating']) &&
parseFloat(movie['kp_rating']) < 10
? parseInt(parseFloat(movie['kp_rating']) * 10 + '')
: parseInt(movie['kp_rating']);
}
if (movie['imdb_rating']) {
movie['imdb_rating'] = ('' + movie['imdb_rating']).replace(
/,/g,
''
);
movie['imdb_rating'] =
parseFloat(movie['imdb_rating']) &&
parseFloat(movie['imdb_rating']) < 10
? parseInt(parseFloat(movie['imdb_rating']) * 10 + '')
: parseInt(movie['imdb_rating']);
}
if (movie['vote']) {
movie['vote'] = ('' + movie['vote']).replace(/,/g, '');
movie['vote'] = parseInt(movie['vote'] + '');
}
if (movie['kp_vote']) {
movie['kp_vote'] = ('' + movie['kp_vote']).replace(/,/g, '');
movie['kp_vote'] = parseInt(movie['kp_vote'] + '');
}
if (movie['imdb_vote']) {
movie['imdb_vote'] = ('' + movie['imdb_vote']).replace(
/,/g,
''
);
movie['imdb_vote'] = parseInt(movie['imdb_vote'] + '');
}
if (movie['premiere']) {
if (/[0-9]{1,2}\s+[а-я]+\s+[0-9]{4}/i.test(movie['premiere'])) {
var p = movie['premiere'].replace(
/([0-9]{1,2})\s+([а-я]+)\s+([0-9]{4})/i,
'$3-$2-$1'
);
[
'января',
'февраля',
'марта',
'апреля',
'мая',
'июня',
'июля',
'августа',
'сентября',
'октября',
'ноября',
'декабря'
].forEach(function(month, i) {
if (p.toLowerCase().indexOf(month) + 1) {
p = p.toLowerCase().replace(month, i + 1 + '');
}
});
p = p
.split('-')
.map(function(num) {
if (num && parseInt(num) < 10) {
num = '0' + num;
}
return num;
})
.join('-');
movie['premiere'] = p;
}
if (/[0-9]{1,2}\s+[a-z]+\s+[0-9]{4}/i.test(movie['premiere'])) {
var pp = movie['premiere'].replace(
/([0-9]{1,2})\s+([a-z]+)\s+([0-9]{4})/i,
'$3-$2-$1'
);
[
'JAN',
'FEB',
'MAR',
'APR',
'MAY',
'JUN',
'JUL',
'AUG',
'SEP',
'OCT',
'NOV',
'DEC'
].forEach(function(month, i) {
if (pp.toUpperCase().indexOf(month) + 1) {
p = p.toUpperCase().replace(month, i + 1 + '');
}
});
p = p
.split('-')
.map(function(num) {
if (num && parseInt(num) < 10) {
num = '0' + num;
}
return num;
})
.join('-');
movie['premiere'] = p;
}
if (/[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}/i.test(movie['premiere'])) {
movie['premiere'] = movie['premiere'] + 'T00:00:00.000Z';
}
var year = new Date(movie['premiere']).getFullYear();
if (!movie['year']) {
movie['year'] = !isNaN(year) ? year + '' : '0';
}
movie['premiere'] = !isNaN(year)
? Math.floor(
new Date(movie['premiere']).getTime() /
1000 /
60 /
60 /
24 +
719528
) + ''
: '0';
}
if (
movie['type'] &&
(movie['type'].toLowerCase().indexOf('1') + 1 ||
movie['type'].toLowerCase().indexOf('tv') + 1 ||
movie['type'].toLowerCase().indexOf('show') + 1 ||
movie['type'].toLowerCase().indexOf('ser') + 1 ||
movie['type'].toLowerCase().indexOf('script') + 1)
) {
movie['type'] = 1;
} else {
movie['type'] = 0;
}
var req_id =
(movie['kp_id'] && parseInt(movie['kp_id'])) ||
(movie['custom'] &&
!movie['type'] &&
movie['custom']['tmdb_id'] &&
parseInt(movie['custom']['tmdb_id']) &&
parseInt(movie['custom']['tmdb_id']) + 200000000) ||
(movie['custom'] &&
movie['type'] &&
movie['custom']['tmdb_id'] &&
parseInt(movie['custom']['tmdb_id']) &&
parseInt(movie['custom']['tmdb_id']) + 300000000) ||
(movie['custom'] &&
movie['custom']['imdb_id'] &&
parseInt(movie['custom']['imdb_id']) &&
parseInt(movie['custom']['imdb_id']) + 400000000) ||
(movie['custom'] &&
movie['custom']['douban_id'] &&
parseInt(movie['custom']['douban_id']) &&
parseInt(movie['custom']['douban_id']) + 600000000) ||
(movie['custom'] &&
movie['custom']['wa_id'] &&
parseInt(movie['custom']['wa_id']) &&
parseInt(movie['custom']['wa_id']) + 700000000) ||
(movie['custom'] &&
movie['custom']['tvmaze_id'] &&
parseInt(movie['custom']['tvmaze_id']) &&
parseInt(movie['custom']['tvmaze_id']) + 800000000) ||
(movie['custom'] &&
movie['custom']['movie_id'] &&
parseInt(movie['custom']['movie_id']) &&
parseInt(movie['custom']['movie_id']) + 900000000);
Not fill info for closed movies
2021-03-03 06:45:28 +01:00
if (
!req_id ||
(req_id &&
Added skip movies
2021-03-04 21:59:29 +01:00
config.movies.skip &&
config.movies.skip.length &&
config.movies.skip.indexOf(req_id + '') !== -1)
Not fill info for closed movies
2021-03-03 06:45:28 +01:00
) {
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
return callback();
}
var queries = [];
if (movie['kp_id'] && parseInt(movie['kp_id'])) {
queries.push({ id: parseInt(movie['kp_id']) + '' });
}
if (
movie['custom'] &&
movie['custom']['tmdb_id'] &&
parseInt(movie['custom']['tmdb_id'])
) {
queries.push({
type: movie['type'],
id: 'custom.tmdb_id',
'custom.tmdb_id':
movie['custom']['tmdb_id'].replace(/[^0-9]/g, '') + ''
});
}
if (
movie['custom'] &&
movie['custom']['imdb_id'] &&
parseInt(movie['custom']['imdb_id'])
) {
queries.push({
id: 'custom.imdb_id',
'custom.imdb_id':
movie['custom']['imdb_id'].replace(/[^0-9]/g, '') + ''
});
}
if (
movie['custom'] &&
movie['custom']['douban_id'] &&
parseInt(movie['custom']['douban_id'])
) {
queries.push({
id: 'custom.douban_id',
'custom.douban_id':
movie['custom']['douban_id'].replace(/[^0-9]/g, '') + ''
});
}
if (
movie['custom'] &&
movie['custom']['tvmaze_id'] &&
parseInt(movie['custom']['tvmaze_id'])
) {
queries.push({
id: 'custom.tvmaze_id',
'custom.tvmaze_id':
movie['custom']['tvmaze_id'].replace(/[^0-9]/g, '') + ''
});
}
if (
movie['custom'] &&
movie['custom']['wa_id'] &&
parseInt(movie['custom']['wa_id'])
) {
queries.push({
id: 'custom.wa_id',
'custom.wa_id':
movie['custom']['wa_id'].replace(/[^0-9]/g, '') + ''
});
}
if (
movie['custom'] &&
movie['custom']['movie_id'] &&
parseInt(movie['custom']['movie_id'])
) {
queries.push({
id: 'custom.movie_id',
'custom.movie_id':
movie['custom']['movie_id'].replace(/[^0-9]/g, '') + ''
});
}
var current_movie = null;
async.eachOfLimit(
queries,
1,
function(query, query_index, callback) {
if (current_movie) {
return callback('STOP');
}
var req = {};
req['certainly'] = true;
var w = '';
if (query.id.indexOf('custom.') + 1 && query[query.id]) {
w = query.id + " = '" + query[query.id] + "'";
w += query.type
? ' AND type = ' +
(query.type.toString() === '1' ? '1' : '0')
: '';
} else {
Fixed string ID
2021-02-14 06:22:51 +01:00
w = '`kp_id` = ' + query.id + '';
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
}
pool.getConnection(function(err, connection) {
if (err) {
connection.release();
return callback();
}
connection.query(
'SELECT * FROM rt_' +
config.domain.replace(/[^a-z0-9]/g, '_') +
' WHERE ' +
w +
' LIMIT 0,1 OPTION max_matches = 1',
function(err, rt) {
connection.release();
if (err) {
console.error(err);
return callback('STOP');
}
if (rt && rt.length) {
current_movie = Object.assign({}, rt[0]);
return callback('STOP');
}
return callback();
}
);
});
},
function() {
if (current_movie) {
delete current_movie['all_movies'];
delete movie['all_movies'];
var cm = Object.assign({}, current_movie);
delete cm['custom'];
var commit_movie = JSON.stringify(
Object.keys(cm)
.sort()
.reduce(function(obj, key) {
obj[key] = cm[key];
return obj;
}, {})
);
var parse_movie = Object.assign({}, movie);
var custom_no_change = false;
Object.keys(parse_movie).forEach(function(k) {
if (!parse_movie[k]) {
delete parse_movie[k];
}
});
Object.keys(current_movie).forEach(function(k) {
if (!current_movie[k] && parse_movie[k]) {
delete current_movie[k];
}
});
if (
(current_movie['poster'] === '1' ||
current_movie['poster'] === 1 ||
current_movie['poster'] === '0' ||
current_movie['poster'] === 0) &&
parse_movie['poster']
) {
delete current_movie['poster'];
}
if (
current_movie['quality'] &&
parse_movie['quality'] &&
current_movie['quality'] !== parse_movie['quality']
) {
delete current_movie['quality'];
}
if (
current_movie['translate'] &&
parse_movie['translate'] &&
current_movie['translate'] !== parse_movie['translate']
) {
delete current_movie['translate'];
}
var current_movie_custom = {};
if (current_movie.custom) {
if (
current_movie.custom &&
typeof current_movie.custom === 'string'
) {
current_movie_custom = JSON.parse(current_movie.custom);
} else {
current_movie_custom = Object.assign(
{},
current_movie.custom
);
}
}
if (current_movie.custom && parse_movie.custom) {
var parse_movie_custom = {};
if (
parse_movie.custom &&
typeof parse_movie.custom === 'string'
) {
parse_movie_custom = JSON.parse(parse_movie.custom);
} else {
parse_movie_custom = Object.assign(
{},
parse_movie.custom
);
}
[1, 2, 3, 4, 5].forEach(function(i) {
if (
current_movie_custom['player' + i] &&
parse_movie_custom['player' + i]
) {
delete current_movie_custom['player' + i];
if (parse_movie_custom['player' + i] === 'none') {
delete parse_movie_custom['player' + i];
}
}
});
[
'imdb_id',
'tmdb_id',
'douban_id',
'tvmaze_id',
'wa_id',
'movie_id'
].forEach(function(id) {
if (
typeof parse_movie_custom[id] !== 'undefined' &&
!parse_movie_custom[id]
) {
delete parse_movie_custom[id];
}
if (
typeof current_movie_custom[id] !== 'undefined' &&
!current_movie_custom[id]
) {
delete current_movie_custom[id];
}
});
Added stop receiving info
2021-02-15 08:25:53 +01:00
if (typeof parse_movie_custom['unique'] !== 'undefined') {
parse_movie_custom['unique'] =
parse_movie_custom['unique'] === true ||
parse_movie_custom['unique'] === 'true';
}
if (
typeof current_movie_custom['unique'] !== 'undefined'
) {
current_movie_custom['unique'] =
current_movie_custom['unique'] === true ||
current_movie_custom['unique'] === 'true';
}
if (
typeof parse_movie_custom['unique'] !== 'undefined' &&
typeof current_movie_custom['unique'] !== 'undefined' &&
parse_movie_custom['unique'] !==
current_movie_custom['unique']
) {
delete current_movie_custom['unique'];
}
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
current_movie.custom = Object.assign(
{},
parse_movie_custom,
current_movie_custom
);
if (
JSON.stringify(
Object.keys(current_movie.custom)
.sort()
.reduce(function(obj, key) {
obj[key] = current_movie.custom[key];
return obj;
}, {})
) ===
JSON.stringify(
Object.keys(current_movie_custom)
.sort()
.reduce(function(obj, key) {
obj[key] = current_movie_custom[key];
return obj;
}, {})
)
) {
custom_no_change = true;
}
}
if (
current_movie_custom &&
typeof current_movie_custom.unique === 'boolean' &&
current_movie_custom.unique === false
) {
if (
current_movie['title_ru'] &&
parse_movie['title_ru'] &&
current_movie['title_ru'] !== parse_movie['title_ru']
) {
delete current_movie['title_ru'];
}
if (
current_movie['title_en'] &&
parse_movie['title_en'] &&
current_movie['title_en'] !== parse_movie['title_en']
) {
delete current_movie['title_en'];
}
if (
current_movie['description'] &&
parse_movie['description'] &&
current_movie['description'] !==
Changed log
2021-03-13 07:13:24 +01:00
parse_movie['description'] &&
current_movie['description'] <
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
parse_movie['description']
) {
delete current_movie['description'];
}
}
var update_movie = Object.assign(
{},
parse_movie,
current_movie
);
[
'year',
'rating',
'vote',
'kp_rating',
'kp_vote',
'imdb_rating',
'imdb_vote',
'premiere'
].forEach(function(attr_uint) {
var current =
(typeof current_movie[attr_uint] !== 'undefined' &&
current_movie[attr_uint] &&
parseFloat(current_movie[attr_uint])) ||
0;
var parse =
(typeof parse_movie[attr_uint] !== 'undefined' &&
parse_movie[attr_uint] &&
parseFloat(parse_movie[attr_uint])) ||
0;
if (
attr_uint === 'rating' ||
attr_uint === 'kp_rating' ||
attr_uint === 'imdb_rating'
) {
if (parse && parse > 100) {
parse = 0;
}
if (!parse) {
return;
}
if (parse < 10) {
parse = parseInt(parse * 10 + '');
}
update_movie[attr_uint] = parse;
return;
}
if (
attr_uint === 'year' ||
attr_uint === 'premiere' ||
attr_uint === 'vote' ||
attr_uint === 'kp_vote' ||
attr_uint === 'imdb_vote'
) {
parse = parseInt(parse + '');
}
if (parse > current) {
update_movie[attr_uint] = parseInt(parse);
}
if (
attr_uint === 'year' &&
typeof update_movie['year'] !== 'undefined'
) {
update_movie['year'] = update_movie['year'] + '';
}
});
['country', 'director', 'genre', 'actor'].forEach(function(
attr_string
) {
var parse =
(typeof parse_movie[attr_string] !== 'undefined' &&
parse_movie[attr_string]) ||
'';
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
var current = (
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
(typeof current_movie[attr_string] !== 'undefined' &&
current_movie[attr_string]) ||
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
''
).replace('_empty', '');
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
if (parse.length > current.length) {
update_movie[attr_string] = parse_movie[attr_string];
}
});
var cm2 = Object.assign({}, update_movie);
delete cm2['custom'];
var commit_movie2 = JSON.stringify(
Object.keys(cm2)
.sort()
.reduce(function(obj, key) {
obj[key] = cm2[key];
return obj;
}, {})
);
if (custom_no_change && commit_movie === commit_movie2) {
console.log(
'[REALTIME]',
id_index + 1,
'/',
ids.length,
iter_req ? '[' + iter_req + ']' : '',
'NO UPDATE',
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
current_movie.id,
'PROXY',
Added detect captcha
2021-02-16 09:21:10 +01:00
movies_proxy || false,
movies_captcha || ''
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
return callback();
}
console.log(
'[REALTIME]',
id_index + 1,
'/',
ids.length,
iter_req ? '[' + iter_req + ']' : '',
'UPDATE MOVIE',
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
current_movie.id,
'PROXY',
movies_proxy || false
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
console.log(update_movie);
CP_save.save(update_movie, 'rt', function(err, result) {
update_m++;
Changed log
2021-03-13 07:13:24 +01:00
if (err) console.log(err);
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
return callback(err);
});
} else {
if (!movie['title_ru'] && !movie['title_en']) {
console.log(
'[REALTIME]',
id_index + 1,
'/',
ids.length,
iter_req ? '[' + iter_req + ']' : '',
'NO SAVE',
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
req_id,
'PROXY',
movies_proxy || false
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
return callback();
}
console.log(
'[REALTIME]',
id_index + 1,
'/',
ids.length,
iter_req ? '[' + iter_req + ']' : '',
'SAVE MOVIE',
Added cookies for parse, added new image source
2021-02-16 09:04:54 +01:00
req_id,
'PROXY',
movies_proxy || false
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
);
console.log(movie);
CP_save.save(movie, 'rt', function(err, result) {
added_m++;
Changed log
2021-03-13 07:13:24 +01:00
if (err) console.log(err);
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
return callback(err);
});
}
}
);
}
},
function() {
console.log('[REALTIME]', 'ADDED:', added_m);
console.log('[REALTIME]', 'UPDATE:', update_m);
return callback();
}
);
}
},
function() {
process.env['NO_CACHE'] = undefined;
Added stop receiving info
2021-02-15 08:25:53 +01:00
console.log(
'[REALTIME]',
new Date()
.toJSON()
.replace('T', ' ')
.split('.')[0],
'[',
Math.ceil(os.freemem()),
'MB ]',
os.loadavg(1).toFixed(2),
os.loadavg(5).toFixed(2),
os.loadavg(15).toFixed(2)
);
console.timeEnd('[REALTIME] DONE');
return process.exit(0);
Fixed many IDs parse, updated GeoLite2
2021-02-13 06:50:08 +01:00
}
);
})();
/**
* Valid JSON.
*
*/
function tryParseJSON(jsonString) {
try {
if (jsonString && typeof jsonString === 'string') {
if (jsonString.indexOf('<?xml') + 1) {
var result = convert.xml2js(jsonString, { compact: true });
if (result && typeof result === 'object') {
return result;
}
}
var o = JSON.parse(jsonString);
if (o && typeof o === 'object') {
return o;
}
} else {
if (jsonString && typeof jsonString === 'object') {
return jsonString;
}
}
} catch (e) {}
return null;
}
Reference in a new issue Copy permalink