Added protection against "too many SQL variables" for SQLite3

no issue

- discovered while testing
- SQLite3 has a restriction that you cannot query more than 999 SQL variables (e.g. fetch all tag relations for all posts)
- if you have more than 999 posts in your database, Ghost wasn't able to finish the bootstrap
  - the url service tried to fetch all posts & relations, but could not complete, because SQlite3 has thrown
    -> "too many SQL variables"
- i have added a recursive query strategy only for SQLite3
  - use offset/limit
This commit is contained in:
kirrg001 2018-08-08 19:17:01 +02:00
parent 6c18ceef25
commit 08f54d2f57
2 changed files with 30 additions and 2 deletions

View File

@ -1071,6 +1071,14 @@ ghostBookshelf.Model = ghostBookshelf.Model.extend({
let query = ghostBookshelf.knex(tableNames[modelName]);
if (options.offset) {
query.offset(options.offset);
}
if (options.limit) {
query.limit(options.limit);
}
// exclude fields if enabled
if (reducedFields) {
const toSelect = _.keys(schema.tables[tableNames[modelName]]);
@ -1090,6 +1098,11 @@ ghostBookshelf.Model = ghostBookshelf.Model.extend({
return query.then((objects) => {
debug('fetched', modelName, filter);
if (!objects.length) {
debug('No more entries found');
return Promise.resolve([]);
}
let props = {};
if (!withRelated) {

View File

@ -2,6 +2,7 @@ const debug = require('ghost-ignition').debug('services:url:resources'),
Promise = require('bluebird'),
_ = require('lodash'),
Resource = require('./Resource'),
config = require('../../config'),
models = require('../../models'),
common = require('../../lib/common');
@ -132,16 +133,30 @@ class Resources {
});
}
_fetch(resourceConfig) {
_fetch(resourceConfig, options = {offset: 0, limit: 999}) {
debug('_fetch', resourceConfig.type, resourceConfig.modelOptions);
return models.Base.Model.raw_knex.fetchAll(resourceConfig.modelOptions)
let modelOptions = _.cloneDeep(resourceConfig.modelOptions);
const isSQLite = config.get('database:client') === 'sqlite3';
// CASE: prevent "too many SQL variables" error on SQLite3
if (isSQLite) {
modelOptions.offset = options.offset;
modelOptions.limit = options.limit;
}
return models.Base.Model.raw_knex.fetchAll(modelOptions)
.then((objects) => {
debug('fetched', resourceConfig.type, objects.length);
_.each(objects, (object) => {
this.data[resourceConfig.type].push(new Resource(resourceConfig.type, object));
});
if (objects.length && isSQLite) {
options.offset = (options.offset + 1) * options.limit;
return this._fetch(resourceConfig, {offset: options.offset, limit: options.limit});
}
});
}