tqt
/
Ghost
mirror of https://github.com/TryGhost/Ghost.git
2
1
Fork 0
Code Issues 1 Releases Wiki Activity

Data models import, export, and reset for 002 

- added line to index.js to set node_env to development if it is not set
 - fixed a small bug with the persistent notifications and used them on debug page from server side
 - added 002 files to manage export and import for 002
 - 002 import is somewhat smarter than 001, merging settings (except version), replacing user & clearing primary keys
 - added reset to models and migration, which does the down operation the same way that init does the up operation
 - import and reset clear session & redirect to login / signup
 - additional unit tests
This commit is contained in:
Hannah Wolfe 2013-08-03 16:11:16 +01:00 committed by ErisDS
parent 2a5e7ad516
commit 338109c762
14 changed files with 540 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Gruntfile.js
View File

@ -114,6 +114,13 @@ var path = require('path'),
perm: {
src: ['core/test/unit/**/permissions_spec.js']
},
migrate: {
src: [
'core/test/unit/**/export_spec.js',
'core/test/unit/**/import_spec.js'
]
}
},
@ -301,6 +308,9 @@ var path = require('path'),
// Run permisisons tests only
grunt.registerTask("test-p", ["mochacli:perm"]);
// Run migrations tests only
grunt.registerTask("test-m", ["mochacli:migrate"]);
// Run tests and lint code
grunt.registerTask("validate", ["jslint", "mochacli:all"]);

3
core/ghost.js
View File

@ -74,6 +74,9 @@ Ghost = function () {
// Holds the plugin directories temporarily
instance.pluginDirectories = {};
// Holds the persistent notifications
instance.notifications = [];
app = express();
polyglot = new Polyglot();

101
core/server/controllers/admin.js
View File

@ -174,15 +174,31 @@ adminControllers = {
})
.otherwise(function (error) {
// Notify of an error if it occurs
req.flash("error", error.message || error);
res.redirect("/ghost/debug");
var notification = {
type: 'error',
message: error.message || error,
status: 'persistent',
id: 'per-' + (ghost.notifications.length + 1)
};
return api.notifications.add(notification).then(function () {
res.redirect("/ghost/debug/");
});
});
},
'import': function (req, res) {
if (!req.files.importfile) {
// Notify of an error if it occurs
req.flash("error", "Must select a file to import");
return res.redirect("/ghost/debug");
var notification = {
type: 'error',
message: "Must select a file to import",
status: 'persistent',
id: 'per-' + (ghost.notifications.length + 1)
};
return api.notifications.add(notification).then(function () {
res.redirect("/ghost/debug/");
});
}
// Get the current version for importing
@ -213,44 +229,59 @@ adminControllers = {
return dataImport(currentVersion, importData);
});
})
.then(function () {
req.flash("success", "Data imported");
})
.otherwise(function (error) {
.then(function importSuccess() {
var notification = {
type: 'success',
message: "Data imported. Log in with the user details you imported",
status: 'persistent',
id: 'per-' + (ghost.notifications.length + 1)
};
return api.notifications.add(notification).then(function () {
delete req.session.user;
res.redirect('/ghost/login/');
});
}, function importFailure(error) {
// Notify of an error if it occurs
req.flash("error", error.message || error);
})
.then(function () {
res.redirect("/ghost/debug");
var notification = {
type: 'error',
message: error.message || error,
status: 'persistent',
id: 'per-' + (ghost.notifications.length + 1)
};
return api.notifications.add(notification).then(function () {
res.redirect('/ghost/debug/');
});
});
},
'reset': function (req, res) {
// Grab the current version so we can get the migration
api.settings.read({ key: "currentVersion" })
.then(function (setting) {
var migration = require("../../shared/data/migration/" + setting.value);
dataProvider.reset()
.then(function resetSuccess() {
var notification = {
type: 'success',
message: "Database reset. Create a new user",
status: 'persistent',
id: 'per-' + (ghost.notifications.length + 1)
};
// Run the downward migration
return migration.down();
}, function () {
// If no version in the DB, assume 001
var migration = require(".././migration/001");
return api.notifications.add(notification).then(function () {
delete req.session.user;
res.redirect('/ghost/signup/');
});
}, function resetFailure(error) {
var notification = {
type: 'error',
message: error.message || error,
status: 'persistent',
id: 'per-' + (ghost.notifications.length + 1)
};
// Run the downward migration
return migration.down();
})
.then(function () {
// Re-initalize the providers (should run the migration up again)
return dataProvider.init();
})
.then(function () {
req.flash("success", "Database reset");
})
.otherwise(function (error) {
req.flash("error", error.message || error);
})
.then(function () {
res.redirect('/ghost/debug');
return api.notifications.add(notification).then(function () {
res.redirect('/ghost/debug/');
});
});
}
}

49
core/server/data/export/002.js Normal file
View File

@ -0,0 +1,49 @@
var _ = require("underscore"),
when = require("when"),
knex = require('../../models/base').Knex,
Exporter002;
Exporter002 = function () {
this.version = "002";
};
Exporter002.prototype.exportData = function () {
var self = this,
tables = [
'posts', 'users', 'roles', 'roles_users', 'permissions',
'permissions_roles', 'settings', 'tags', 'posts_tags',
'custom_data', 'posts_custom_data'
],
selectOps = _.map(tables, function (name) {
return knex(name).select();
});
return when.all(selectOps).then(function (tableData) {
var exportData = {
meta: {
exported_on: new Date().getTime(),
version: self.version
},
data: {
// Filled below
}
};
_.each(tables, function (name, i) {
exportData.data[name] = tableData[i];
});
return when.resolve(exportData);
}, function (err) {
console.log("Error exporting data: " + err);
});
};
module.exports = {
// Make available for unit tests
Exporter002: Exporter002,
exportData: function () {
return new Exporter002().exportData();
}
};

9
core/server/data/export/index.js
View File

@ -1,8 +1,13 @@
var when = require('when');
var when = require('when'),
migration = require('../migration');
module.exports = function (version) {
var exporter;
if (version > migration.currentVersion) {
return when.reject("Your data version is ahead of the current Ghost version. Please upgrade in order to export.");
}
try {
exporter = require("./" + version);
} catch (ignore) {
@ -10,7 +15,7 @@ module.exports = function (version) {
}
if (!exporter) {
return when.reject("No exporter found");
return when.reject("No exporter found for data version " + version);
}
return exporter.exportData();

116
core/server/data/import/002.js Normal file
View File

@ -0,0 +1,116 @@
var when = require("when"),
_ = require("underscore"),
knex = require('../../models/base').Knex,
errors = require('../../errorHandling'),
Importer002;
function stripProperties(properties, data) {
_.each(data, function (obj) {
_.each(properties, function (property) {
delete obj[property];
});
});
return data;
}
Importer002 = function () {
_.bindAll(this, "basicImport");
this.version = "002";
this.importFrom = {
"001": this.basicImport,
"002": this.basicImport
};
};
Importer002.prototype.importData = function (data) {
return this.canImport(data)
.then(function (importerFunc) {
return importerFunc(data);
}, function (reason) {
return when.reject(reason);
});
};
Importer002.prototype.canImport = function (data) {
if (data.meta && data.meta.version && this.importFrom[data.meta.version]) {
return when.resolve(this.importFrom[data.meta.version]);
}
return when.reject("Unsupported version of data");
};
// No data needs modifying, we just import whatever tables are available
Importer002.prototype.basicImport = function (data) {
var ops = [];
_.each(data.data, function (tableData, name) {
switch (name) {
case 'posts':
// we want to import all posts as new posts for now
// TODO: eventually we should be smart about posts which have the same title & content
// so that we don't create duplicates
if (tableData && tableData.length) {
tableData = stripProperties(['id'], tableData);
ops.push(knex(name).insert(tableData));
}
break;
case 'users':
// the current data model should only ever have one user.
// So we update the current one with the first one from the imported data
if (tableData && tableData.length) {
tableData = stripProperties(['id'], tableData);
ops.push(knex(name).where('id', 1)
.update(tableData[0]));
}
break;
case 'settings':
// for settings we need to update individual settings, and insert any missing ones
// the one setting we MUST NOT update is the currentVersion settings
var blackList = ['currentVersion'];
if (tableData && tableData.length) {
tableData = stripProperties(['id'], tableData);
_.each(tableData, function (data) {
if (blackList.indexOf(data.key) === -1) {
ops.push(knex(name).where('key', data.key)
.update(data).then(function (success) {
// if no lines were updated then we need to insert instead
return success === 0 ? knex(name).insert(data) : when.resolve(success);
}));
}
});
}
break;
case 'permissions':
case 'roles':
case 'permissions_roles':
case 'permissions_users':
case 'roles_users':
// do nothing with these tables, the data shouldn't have changed from the fixtures
break;
default:
// any other tables, if they have data, remove the primary key and insert it
if (tableData && tableData.length) {
tableData = stripProperties(['id'], tableData);
ops.push(knex(name).insert(tableData));
}
break;
}
});
return when.all(ops).then(function (results) {
return when.resolve(results);
}, function (err) {
return when.reject("Error importing data: " + err.message || err, err.stack);
});
};
module.exports = {
Importer002: Importer002,
importData: function (data) {
return new Importer002().importData(data);
}
};

1
core/server/data/migration/002.js
View File

@ -60,7 +60,6 @@ up = function () {
]).then(function () {
// Once we create all of the initial tables, bootstrap any of the data
return when.all([
knex('settings').insert(fixtures.settings)
]);

120
core/server/data/migration/index.js
View File

@ -9,44 +9,64 @@ var _ = require('underscore'),
// we could probably load it from the config file.
currentVersion = "002";
function getCurrentVersion() {
return knex.Schema.hasTable('settings').then(function () {
// Check for the current version from the settings table
return knex('settings')
.where('key', 'currentVersion')
.select('value')
.then(function (currentVersionSetting) {
if (currentVersionSetting && currentVersionSetting.length > 0) {
currentVersionSetting = currentVersionSetting[0].value;
} else {
// we didn't get a response we understood, assume initialVersion
currentVersionSetting = initialVersion;
}
return currentVersionSetting;
});
});
}
module.exports = {
currentVersion: currentVersion,
// Check for whether data is needed to be bootstrapped or not
init: function () {
var that = this;
var self = this;
return knex.Schema.hasTable('settings').then(function () {
// Check for the current version from the settings table
return knex('settings')
.where('key', 'currentVersion')
.select('value')
.then(function (currentVersionSetting) {
if (currentVersionSetting && currentVersionSetting.length > 0) {
currentVersionSetting = currentVersionSetting[0].value;
} else {
// we didn't get a response we understood, assume initialVersion
currentVersionSetting = initialVersion;
}
// We are assuming here that the currentVersionSetting will
// always be less than the currentVersion value.
if (currentVersionSetting === currentVersion) {
return when.resolve();
}
// Bring the data up to the latest version
return that.migrateFromVersion(currentVersion);
}, errors.logAndThrowError);
return getCurrentVersion().then(function (currentVersionSetting) {
// We are assuming here that the currentVersionSetting will
// always be less than the currentVersion value.
if (currentVersionSetting === currentVersion) {
return when.resolve();
}
// Bring the data up to the latest version
return self.migrateUpFromVersion(currentVersion);
}, function () {
// If the settings table doesn't exist, bring everything up from initial version.
return that.migrateFromVersion(initialVersion);
return self.migrateUpFromVersion(initialVersion);
});
},
// ### Reset
// Migrate from where we are down to nothing.
reset: function () {
var self = this;
return getCurrentVersion().then(function (currentVersionSetting) {
// bring everything down from the current version
return self.migrateDownFromVersion(currentVersionSetting);
}, function () {
// If the settings table doesn't exist, bring everything down from initial version.
return self.migrateDownFromVersion(initialVersion);
});
},
// Migrate from a specific version to the latest
migrateFromVersion: function (version) {
migrateUpFromVersion: function (version, max) {
var versions = [],
maxVersion = this.getVersionAfter(currentVersion),
maxVersion = max || this.getVersionAfter(currentVersion),
currVersion = version,
tasks = [];
@ -73,6 +93,35 @@ module.exports = {
return series(tasks);
},
migrateDownFromVersion: function (version) {
var versions = [],
minVersion = this.getVersionBefore(initialVersion),
currVersion = version,
tasks = [];
// Aggregate all the versions we need to do migrations for
while (currVersion !== minVersion) {
versions.push(currVersion);
currVersion = this.getVersionBefore(currVersion);
}
// Aggregate all the individual up calls to use in the series(...) below
tasks = _.map(versions, function (taskVersion) {
return function () {
try {
var migration = require('./' + taskVersion);
return migration.down();
} catch (e) {
errors.logError(e);
return when.reject(e);
}
};
});
// Run each migration in series
return series(tasks);
},
// Get the following version based on the current
getVersionAfter: function (currVersion) {
@ -93,5 +142,24 @@ module.exports = {
}
return nextVersion;
},
getVersionBefore: function (currVersion) {
var currVersionNum = parseInt(currVersion, 10),
prevVersion;
if (isNaN(currVersionNum)) {
currVersionNum = parseInt(initialVersion, 10);
}
currVersionNum -= 1;
prevVersion = String(currVersionNum);
// Pad with 0's until 3 digits
while (prevVersion.length < 3) {
prevVersion = "0" + prevVersion;
}
return prevVersion;
}
};

5
core/server/models/index.js
View File

@ -8,5 +8,10 @@ module.exports = {
Settings: require('./settings').Settings,
init: function () {
return migrations.init();
},
reset: function () {
return migrations.reset().then(function () {
return migrations.init();
});
}
};

3
core/server/views/debug.hbs
View File

@ -15,7 +15,6 @@
<h2 class="title">General</h2>
</header>
<section class="content">
{{> flashes}}
<form id="settings-export">
<fieldset>
<label>
@ -31,7 +30,7 @@
<b>Import</b>
<input type="file" class="button-add" name="importfile"></input>
<input type="submit" class="button-save" value="Import"></input>
<p>Import from another Ghost installation.</p>
<p>Import from another Ghost installation. If you import a user, this will replace the current user & log you out.</p>
</label>
</fieldset>
</form>

60
core/test/unit/export_spec.js
View File

@ -4,8 +4,10 @@ var _ = require("underscore"),
when = require('when'),
sinon = require('sinon'),
helpers = require('./helpers'),
migration = require('../../server/data/migration'),
exporter = require('../../server/data/export'),
Exporter001 = require('../../server/data/export/001'),
Exporter002 = require('../../server/data/export/002'),
errors = require('../../server/errorHandling');
describe("Export", function () {
@ -13,7 +15,8 @@ describe("Export", function () {
should.exist(exporter);
beforeEach(function (done) {
helpers.resetData().then(function () {
// clear database... we need to initialise it manually for each test
helpers.clearData().then(function () {
done();
}, done);
});
@ -37,7 +40,11 @@ describe("Export", function () {
should.exist(Exporter001);
it("exports data", function (done) {
exporter("001").then(function (exportData) {
// initialise database to version 001 - confusingly we have to set the max version to be one higher
// than the migration version we want
migration.migrateUpFromVersion('001', '002').then(function () {
return exporter("001");
}).then(function (exportData) {
var tables = ['posts', 'users', 'roles', 'roles_users', 'permissions', 'permissions_roles', 'settings'];
should.exist(exportData);
@ -46,6 +53,55 @@ describe("Export", function () {
should.exist(exportData.data);
exportData.meta.version.should.equal("001");
_.findWhere(exportData.data.settings, {key: "currentVersion"}).value.should.equal("001");
_.each(tables, function (name) {
should.exist(exportData.data[name]);
});
// 002 data should not be present
should.not.exist(exportData.data.tags);
done();
}).then(null, done);
});
});
it("resolves 002", function (done) {
var exportStub = sinon.stub(Exporter002, "exportData", function () {
return when.resolve();
});
exporter("002").then(function () {
exportStub.called.should.equal(true);
exportStub.restore();
done();
}).then(null, done);
});
describe("002", function () {
should.exist(Exporter001);
it("exports data", function (done) {
// initialise database to version 001 - confusingly we have to set the max version to be one higher
// than the migration version we want
migration.migrateUpFromVersion('001', '003').then(function () {
return exporter("002");
}).then(function (exportData) {
var tables = [
'posts', 'users', 'roles', 'roles_users', 'permissions', 'permissions_roles',
'settings', 'tags', 'posts_tags', 'custom_data', 'posts_custom_data'
];
should.exist(exportData);
should.exist(exportData.meta);
should.exist(exportData.data);
exportData.meta.version.should.equal("002");
_.findWhere(exportData.data.settings, {key: "currentVersion"}).value.should.equal("002");
_.each(tables, function (name) {
should.exist(exportData.data[name]);

10
core/test/unit/helpers.js
View File

@ -3,9 +3,7 @@ process.env.NODE_ENV = process.env.TRAVIS ? 'travis' : 'testing';
var knex = require('../../server/models/base').Knex,
when = require('when'),
migrations = {
one: require("../../server/data/migration/001")
},
migration = require("../../server/data/migration/"),
helpers,
samplePost,
sampleUser,
@ -47,14 +45,16 @@ helpers = {
resetData: function () {
return this.clearData().then(function () {
return migrations.one.up();
return migration.init();
});
},
clearData: function () {
return migrations.one.down();
// we must always try to delete all tables
return migration.migrateDownFromVersion(migration.currentVersion);
},
insertMorePosts: function () {
var lang, status, posts, promises = [], i, j;
for (i = 0; i < 2; i += 1) {

131
core/test/unit/import_spec.js
View File

@ -5,9 +5,11 @@ var _ = require("underscore"),
sinon = require('sinon'),
knex = require("../../server/models/base").Knex,
helpers = require('./helpers'),
migration = require('../../server/data/migration'),
exporter = require('../../server/data/export'),
importer = require('../../server/data/import'),
Importer001 = require('../../server/data/import/001'),
Importer002 = require('../../server/data/import/002'),
errors = require('../../server/errorHandling');
describe("Import", function () {
@ -16,7 +18,8 @@ describe("Import", function () {
should.exist(importer);
beforeEach(function (done) {
helpers.resetData().then(function () {
// clear database... we need to initialise it manually for each test
helpers.clearData().then(function () {
done();
}, done);
});
@ -43,12 +46,21 @@ describe("Import", function () {
it("imports data from 001", function (done) {
var exportData;
// TODO: Should have static test data here?
exporter("001").then(function (exported) {
// initialise database to version 001 - confusingly we have to set the max version to be one higher
// than the migration version we want
migration.migrateUpFromVersion('001', '002').then(function () {
// export the version 001 data ready to import
// TODO: Should have static test data here?
return exporter("001");
}).then(function (exported) {
exportData = exported;
// Clear the data from all tables.
var tables = ['posts', 'users', 'roles', 'roles_users', 'permissions', 'permissions_roles', 'settings'],
// Version 001 exporter required the database be empty...
var tables = [
'posts', 'users', 'roles', 'roles_users', 'permissions', 'permissions_roles',
'settings'
],
truncateOps = _.map(tables, function (name) {
return knex(name).truncate();
});
@ -68,9 +80,114 @@ describe("Import", function () {
should.exist(importedData);
importedData.length.should.equal(3);
importedData[0].length.should.equal(exportData.data.users.length);
// we always have 0 users as there isn't one in fixtures
importedData[0].length.should.equal(0);
importedData[1].length.should.equal(exportData.data.posts.length);
importedData[2].length.should.equal(exportData.data.settings.length);
// version 001 settings have 7 fields
importedData[2].length.should.equal(7);
_.findWhere(exportData.data.settings, {key: "currentVersion"}).value.should.equal("001");
done();
}).then(null, done);
});
});
it("resolves 002", function (done) {
var importStub = sinon.stub(Importer002, "importData", function () {
return when.resolve();
}),
fakeData = { test: true };
importer("002", fakeData).then(function () {
importStub.calledWith(fakeData).should.equal(true);
importStub.restore();
done();
}).then(null, done);
});
describe("002", function () {
this.timeout(4000);
should.exist(Importer002);
it("imports data from 001", function (done) {
var exportData;
// initialise database to version 001 - confusingly we have to set the max version to be one higher
// than the migration version we want
migration.migrateUpFromVersion('001', '002').then(function () {
// export the version 001 data ready to import
// TODO: Should have static test data here?
return exporter("001");
}).then(function (exported) {
exportData = exported;
// now migrate up to the proper version ready for importing - confusingly we have to set the max version
// to be one higher than the migration version we want
return migration.migrateUpFromVersion('002', '003');
}).then(function () {
return importer("002", exportData);
}).then(function () {
// Grab the data from tables
return when.all([
knex("users").select(),
knex("posts").select(),
knex("settings").select()
]);
}).then(function (importedData) {
should.exist(importedData);
importedData.length.should.equal(3);
// we always have 0 users as there isn't one in fixtures
importedData[0].length.should.equal(0);
// import no longer requires all data to be dropped, and adds posts
importedData[1].length.should.equal(exportData.data.posts.length + 1);
// version 002 settings have 10 fields, and settings get updated not inserted
importedData[2].length.should.equal(10);
_.findWhere(importedData[2], {key: "currentVersion"}).value.should.equal("002");
done();
}).then(null, done);
});
it("imports data from 002", function (done) {
var exportData;
// initialise database to version 001 - confusingly we have to set the max version to be one higher
// than the migration version we want
migration.migrateUpFromVersion('001', '003').then(function () {
// export the version 002 data ready to import
// TODO: Should have static test data here?
return exporter("002");
}).then(function (exported) {
exportData = exported;
return importer("002", exportData);
}).then(function () {
// Grab the data from tables
return when.all([
knex("users").select(),
knex("posts").select(),
knex("settings").select()
]);
}).then(function (importedData) {
should.exist(importedData);
importedData.length.should.equal(3);
// we always have 0 users as there isn't one in fixtures
importedData[0].length.should.equal(0);
// import no longer requires all data to be dropped, and adds posts
importedData[1].length.should.equal(exportData.data.posts.length + 1);
// version 002 settings have 10 fields, and settings get updated not inserted
importedData[2].length.should.equal(10);
_.findWhere(importedData[2], {key: "currentVersion"}).value.should.equal("002");
done();
}).then(null, done);

3
index.js
View File

@ -1,4 +1,5 @@
// # Ghost main app file
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
// Module dependencies.
var express = require('express'),
@ -176,7 +177,7 @@ when.all([ghost.init(), filters.loadCoreFilters(ghost), helpers.loadCoreHelpers(
ghost.app().get('/ghost/editor', auth, admin.editor);
ghost.app().get('/ghost/content', auth, admin.content);
ghost.app().get('/ghost/settings*', auth, admin.settings);
ghost.app().get('/ghost/debug', auth, admin.debug.index);
ghost.app().get('/ghost/debug/', auth, admin.debug.index);
ghost.app().get('/ghost/debug/db/export/', auth, admin.debug['export']);
ghost.app().post('/ghost/debug/db/import/', auth, admin.debug['import']);
ghost.app().get('/ghost/debug/db/reset/', auth, admin.debug.reset);