tqt
/
Ghost
mirror of https://github.com/TryGhost/Ghost.git
2
1
Fork 0
Code Issues 1 Releases Wiki Activity

Moved knex-migrator execution into Ghost 

refs #9742, refs https://github.com/TryGhost/Ghost-CLI/issues/759

- required a reordering of Ghost's bootstrap file, because:
  - we have to ensure that no database queries are executed within Ghost during the migrations
  - make 3 sections: check if db needs initialisation, bootstrap Ghost with minimal components (db/models, express apps, load settings+theme)
- create a new `migrator` utility, which tells you which state your db is in and offers an API to execute knex-migrator based on this state
- ensure we still detect an incompatible db: you connect your 2.0 blog with a 0.11 database
- enable maintenance mode if migrations are missing
- if the migration have failed, knex-migrator roll auto rollback
  - you can automatically switch to 1.0 again
- added socket communication for the CLI
This commit is contained in:
kirrg001 2018-07-17 18:10:05 +02:00 committed by Katharina Irrgang
parent 90b56f925a
commit 23b4fd26c6
9 changed files with 306 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/server/adapters/scheduling/SchedulingDefault.js
View File

@ -234,6 +234,14 @@ SchedulingDefault.prototype._pingUrl = function (object) {
object.tries = tries + 1;
self._pingUrl(object);
}, self.retryTimeoutInMs);
common.logging.error(new common.errors.GhostError({
err: err,
context: 'Retrying...',
level: 'normal'
}));
return;
}
common.logging.error(new common.errors.GhostError({

40
core/server/data/db/health.js
View File

@ -1,40 +0,0 @@
var KnexMigrator = require('knex-migrator'),
config = require('../../config'),
common = require('../../lib/common'),
models = require('../../models');
module.exports.check = function healthCheck() {
var knexMigrator = new KnexMigrator({
knexMigratorFilePath: config.get('paths:appRoot')
});
return knexMigrator.isDatabaseOK()
.catch(function (outerErr) {
if (outerErr.code === 'DB_NOT_INITIALISED') {
throw outerErr;
}
// CASE: migration table does not exist, figure out if database is compatible
return models.Settings.findOne({key: 'databaseVersion', context: {internal: true}})
.then(function (response) {
// CASE: no db version key, database is compatible
if (!response) {
throw outerErr;
}
throw new common.errors.DatabaseVersionError({
message: 'Your database version is not compatible with Ghost 1.0.0 (master branch)',
context: 'Want to keep your DB? Use Ghost < 1.0.0 or the "stable" branch. Otherwise please delete your DB and restart Ghost.',
help: 'More information on the Ghost 1.0.0 at https://docs.ghost.org/v1/docs/introduction'
});
})
.catch(function (err) {
// CASE: settings table does not exist
if (err.errno === 1 || err.errno === 1146) {
throw outerErr;
}
throw err;
});
});
};

72
core/server/data/db/migrator.js Normal file
View File

@ -0,0 +1,72 @@
const KnexMigrator = require('knex-migrator'),
config = require('../../config'),
common = require('../../lib/common'),
knexMigrator = new KnexMigrator({
knexMigratorFilePath: config.get('paths:appRoot')
});
module.exports.getState = () => {
let state, err;
return knexMigrator.isDatabaseOK()
.then(() => {
state = 1;
return state;
})
.catch((_err) => {
err = _err;
// CASE: database was never created
if (err.code === 'DB_NOT_INITIALISED') {
state = 2;
return state;
}
// CASE: you have created the database on your own, you have an existing none compatible db?
if (err.code === 'MIGRATION_TABLE_IS_MISSING') {
state = 3;
return state;
}
// CASE: database needs migrations
if (err.code === 'DB_NEEDS_MIGRATION') {
state = 4;
return state;
}
// CASE: database connection errors, unknown cases
throw err;
});
};
module.exports.dbInit = () => {
return knexMigrator.init();
};
module.exports.migrate = () => {
return knexMigrator.migrate();
};
module.exports.isDbCompatible = (connection) => {
return connection.raw('SELECT `key` FROM settings WHERE `key`="databaseVersion";')
.then((response) => {
if (!response || !response[0].length) {
return;
}
throw new common.errors.DatabaseVersionError({
message: 'Your database version is not compatible with Ghost 2.0.',
help: 'Want to keep your DB? Use Ghost < 1.0.0 or the "0.11" branch.' +
'\n\n\n' +
'Want to migrate Ghost 0.11 to 2.0? Please visit https://docs.ghost.org/v1/docs/migrating-to-ghost-1-0-0'
});
})
.catch((err) => {
// CASE settings table doesn't exists
if (err.errno === 1146 || err.errno === 1) {
return;
}
throw err;
});
};

39
core/server/data/migrations/hooks/init/shutdown.js
View File

@ -1,27 +1,20 @@
var _ = require('lodash'),
config = require('../../../../config'),
const _ = require('lodash'),
database = require('../../../db');
module.exports = function after() {
// do not close database connection in test mode, because all tests are executed one after another
// this check is not nice, but there is only one other solution i can think of:
// forward a custom object to knex-migrator, which get's forwarded to the hooks
if (config.get('env').match(/testing/g)) {
return;
module.exports = function shutdown(options = {}) {
if (options.executedFromShell === true) {
// running knex-migrator migrate --init in the shell does two different migration calls within a single process
// we have to ensure that we clear the Ghost cache afterwards, otherwise we operate on a destroyed connection
_.each(require.cache, function (val, key) {
if (key.match(/core\/server/)) {
delete require.cache[key];
}
});
/**
* We have to close Ghost's db connection if knex-migrator was used in the shell.
* Otherwise the process doesn't exit.
*/
return database.knex.destroy();
}
// running knex-migrator migrate --init does two different migration calls within a single process
// we have to ensure that we clear the Ghost cache afterwards, otherwise we operate on a destroyed connection
_.each(require.cache, function (val, key) {
if (key.match(/core\/server/)) {
delete require.cache[key];
}
});
// we need to close the database connection
// the after hook signals the last step of a knex-migrator command
// Example:
// Ghost-CLI calls knexMigrator.init and afterwards it starts Ghost, but Ghost-CLI can't shutdown
// if Ghost keeps a connection alive
return database.knex.destroy();
};

23
core/server/data/migrations/hooks/migrate/shutdown.js
View File

@ -1,18 +1,11 @@
var config = require('../../../../config'),
database = require('../../../db');
const database = require('../../../db');
module.exports = function after() {
// do not close database connection in test mode, because all tests are executed one after another
// this check is not nice, but there is only one other solution i can think of:
// forward a custom object to knex-migrator, which get's forwarded to the hooks
if (config.get('env').match(/testing/g)) {
return;
module.exports = function shutdown(options = {}) {
/**
* We have to close Ghost's db connection if knex-migrator was used in the shell.
* Otherwise the process doesn't exit.
*/
if (options.executedFromShell === true) {
return database.knex.destroy();
}
// we need to close the database connection
// the after hook signals the last step of a knex-migrator command
// Example:
// Ghost-CLI calls knexMigrator.init and afterwards it starts Ghost, but Ghost-CLI can't shutdown
// if Ghost keeps a connection alive
return database.knex.destroy();
};

14
core/server/ghost-server.js
View File

@ -94,7 +94,13 @@ GhostServer.prototype.start = function (externalApp) {
self.httpServer.on('connection', self.connection.bind(self));
self.httpServer.on('listening', function () {
debug('...Started');
common.events.emit('server.start');
// CASE: there are components which listen on this event to initialise after the server has started (in background)
// we want to avoid that they bootstrap during maintenance
if (config.get('maintenance:enabled') === false) {
common.events.emit('server.start');
}
self.logStartMessages();
resolve(self);
});
@ -156,6 +162,8 @@ GhostServer.prototype.hammertime = function () {
GhostServer.prototype.connection = function (socket) {
var self = this;
this.socket = socket;
self.connectionId += 1;
socket._ghostId = self.connectionId;
@ -166,6 +174,10 @@ GhostServer.prototype.connection = function (socket) {
self.connections[socket._ghostId] = socket;
};
GhostServer.prototype.getSocket = function getSocket() {
return this.socket;
};
/**
* ### Close Connections
* Most browsers keep a persistent connection open to the server, which prevents the close callback of

288
core/server/index.js
View File

@ -1,9 +1,6 @@
// # Bootup
// This file needs serious love & refactoring
/**
* make sure overrides get's called first!
* - keeping the overrides require here works for installing Ghost as npm!
* - keeping the overrides import here works for installing Ghost as npm!
*
* the call order is the following:
* - root index requires core module
@ -12,115 +9,200 @@
*/
require('./overrides');
// Module dependencies
var debug = require('ghost-ignition').debug('boot:init'),
config = require('./config'),
Promise = require('bluebird'),
common = require('./lib/common'),
models = require('./models'),
permissions = require('./services/permissions'),
auth = require('./services/auth'),
dbHealth = require('./data/db/health'),
GhostServer = require('./ghost-server'),
scheduling = require('./adapters/scheduling'),
settings = require('./services/settings'),
themes = require('./services/themes'),
urlService = require('./services/url'),
const debug = require('ghost-ignition').debug('boot:init');
const Promise = require('bluebird');
const config = require('./config');
const common = require('./lib/common');
const migrator = require('./data/db/migrator');
const urlService = require('./services/url');
let parentApp;
// Services that need initialisation
apps = require('./services/apps'),
xmlrpc = require('./services/xmlrpc'),
slack = require('./services/slack'),
webhooks = require('./services/webhooks');
function initialiseServices() {
const permissions = require('./services/permissions'),
auth = require('./services/auth'),
apps = require('./services/apps'),
xmlrpc = require('./services/xmlrpc'),
slack = require('./services/slack'),
webhooks = require('./services/webhooks'),
scheduling = require('./adapters/scheduling');
// ## Initialise Ghost
function init() {
debug('Init Start...');
debug('`initialiseServices` Start...');
var ghostServer, parentApp;
// Initialize default internationalization, just for core now
// (settings for language and theme not yet available here)
common.i18n.init();
debug('Default i18n done for core');
models.init();
debug('models done');
return dbHealth.check().then(function () {
debug('DB health check done');
// Populate any missing default settings
// Refresh the API settings cache
return settings.init();
}).then(function () {
debug('Update settings cache done');
common.events.emit('db.ready');
// Full internationalization for core could be here
// in a future version with backend translations
// (settings for language and theme available here;
// internationalization for theme is done
// shortly after, when activating the theme)
//
return Promise.join(
// Initialize the permissions actions and objects
return permissions.init();
}).then(function () {
debug('Permissions done');
return Promise.join(
themes.init(),
// Initialize xmrpc ping
xmlrpc.listen(),
// Initialize slack ping
slack.listen(),
// Initialize webhook pings
webhooks.listen()
);
}).then(function () {
debug('Apps, XMLRPC, Slack done');
// Setup our collection of express apps
parentApp = require('./web/parent-app')();
// Initialise analytics events
if (config.get('segment:key')) {
require('./analytics-events').init();
}
debug('Express Apps done');
}).then(function () {
/**
* @NOTE:
*
* Must happen after express app bootstrapping, because we need to ensure that all
* routers are created and are now ready to register additional routes. In this specific case, we
* are waiting that the AppRouter was instantiated. And then we can register e.g. amp if enabled.
*
* If you create a published post, the url is always stronger than any app url, which is equal.
*/
return apps.init();
}).then(function () {
parentApp.use(auth.init());
debug('Auth done');
return new GhostServer(parentApp);
}).then(function (_ghostServer) {
ghostServer = _ghostServer;
// scheduling can trigger api requests, that's why we initialize the module after the ghost server creation
// scheduling module can create x schedulers with different adapters
debug('Server done');
return scheduling.init({
permissions.init(),
xmlrpc.listen(),
slack.listen(),
webhooks.listen(),
apps.init(),
scheduling.init({
schedulerUrl: config.get('scheduling').schedulerUrl,
active: config.get('scheduling').active,
apiUrl: urlService.utils.urlFor('api', true),
internalPath: config.get('paths').internalSchedulingPath,
contentPath: config.getContentPath('scheduling')
});
})
).then(function () {
debug('XMLRPC, Slack, Webhooks, Apps, Scheduling, Permissions done');
// Initialise analytics events
if (config.get('segment:key')) {
require('./analytics-events').init();
}
}).then(function () {
debug('Scheduling done');
debug('...Init End');
return ghostServer;
parentApp.use(auth.init());
debug('Auth done');
debug('...`initialiseServices` End');
});
}
module.exports = init;
/**
* - initialise models
* - initialise i18n
* - load all settings into settings cache (almost every component makes use of this cache)
* - load active theme
* - create our express apps (site, admin, api)
* - start the ghost server
* - enable maintenance mode if migrations are missing
*/
const minimalRequiredSetupToStartGhost = (dbState) => {
const settings = require('./services/settings');
const models = require('./models');
const themes = require('./services/themes');
const GhostServer = require('./ghost-server');
let ghostServer;
// Initialize Ghost core internationalization
common.i18n.init();
debug('Default i18n done for core');
models.init();
debug('Models done');
return settings.init()
.then(() => {
debug('Settings done');
return themes.init();
})
.then(() => {
debug('Themes done');
parentApp = require('./web/parent-app')();
debug('Express Apps done');
return new GhostServer(parentApp);
})
.then((_ghostServer) => {
ghostServer = _ghostServer;
// CASE: all good or db was just initialised
if (dbState === 1 || dbState === 2) {
common.events.emit('db.ready');
return initialiseServices()
.then(() => {
// CASE: IPC communication to the CLI via child process.
if (process.send) {
process.send({
started: true
});
}
// CASE: Socket communication to the CLI. CLI started Ghost via systemd.
if (ghostServer.getSocket()) {
ghostServer.getSocket().write(JSON.stringify({
started: true
}));
}
return ghostServer;
});
}
// CASE: migrations required, put blog into maintenance mode
if (dbState === 4) {
common.logging.info('Blog is in maintenance mode.');
config.set('maintenance:enabled', true);
migrator.migrate()
.then(() => {
common.events.emit('db.ready');
return initialiseServices();
})
.then(() => {
common.events.emit('server.start');
config.set('maintenance:enabled', false);
common.logging.info('Blog is out of maintenance mode.');
if (process.send) {
process.send({
started: true
});
}
// CASE: Socket communication to the CLI. CLI started Ghost via systemd.
if (ghostServer.getSocket()) {
ghostServer.getSocket().write(JSON.stringify({
started: true
}));
}
})
.catch((err) => {
if (process.send) {
process.send({
started: false,
error: err.message
});
}
// CASE: Socket communication to the CLI. CLI started Ghost via systemd.
if (ghostServer.getSocket()) {
ghostServer.getSocket().write(JSON.stringify({
started: false,
error: err.message
}));
}
common.logging.error(err);
process.exit(-1);
});
return ghostServer;
}
});
};
/**
* Connect to database.
* Check db state.
*/
const isDatabaseInitialisationRequired = () => {
const db = require('./data/db/connection');
let dbState;
return migrator.getState()
.then((state) => {
dbState = state;
// CASE: db initialisation required, wait till finished
if (dbState === 2) {
return migrator.dbInit();
}
// CASE: is db incompatible? e.g. you can't connect a 0.11 database with Ghost 1.0 or 2.0
if (dbState === 3) {
return migrator.isDbCompatible(db)
.then(() => {
dbState = 2;
return migrator.dbInit();
});
}
})
.then(() => {
return minimalRequiredSetupToStartGhost(dbState);
});
};
module.exports = isDatabaseInitialisationRequired;

2
package.json
View File

@ -66,7 +66,7 @@
"js-yaml": "3.12.0",
"jsonpath": "1.0.0",
"knex": "0.14.6",
"knex-migrator": "3.1.6",
"knex-migrator": "3.1.8",
"lodash": "4.17.10",
"markdown-it": "8.4.1",
"markdown-it-footnote": "3.0.1",

6
yarn.lock
View File

@ -3316,9 +3316,9 @@ klaw@^1.0.0:
optionalDependencies:
graceful-fs "^4.1.9"
knex-migrator@3.1.6:
version "3.1.6"
resolved "https://registry.yarnpkg.com/knex-migrator/-/knex-migrator-3.1.6.tgz#480928a060595045acd06253e245920a7bc9544b"
knex-migrator@3.1.8:
version "3.1.8"
resolved "https://registry.yarnpkg.com/knex-migrator/-/knex-migrator-3.1.8.tgz#e1674b85834584d4199748c73c147e95e10935cc"
dependencies:
bluebird "^3.4.6"
commander "2.15.1"