2
1
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2023-12-13 21:00:40 +01:00

Merge pull request #4155 from jgable/importDuplicates

Remove duplicate posts and tags on import
This commit is contained in:
Hannah Wolfe 2014-11-03 15:03:44 +00:00
commit fe659117bc
4 changed files with 936 additions and 9 deletions

View file

@ -7,6 +7,7 @@ var Promise = require('bluebird'),
tables = require('../schema').tables,
validate,
handleErrors,
checkDuplicateAttributes,
sanitize,
cleanError;
@ -65,20 +66,98 @@ handleErrors = function handleErrors(errorList) {
return Promise.reject(processedErrors);
};
sanitize = function sanitize(data) {
// Check for correct UUID and fix if neccessary
_.each(_.keys(data.data), function (tableName) {
_.each(data.data[tableName], function (importValues) {
var uuidMissing = (!importValues.uuid && tables[tableName].uuid) ? true : false,
uuidMalformed = (importValues.uuid && !validator.isUUID(importValues.uuid)) ? true : false;
checkDuplicateAttributes = function checkDuplicateAttributes(data, comparedValue, attribs) {
// Check if any objects in data have the same attribute values
return _.find(data, function (datum) {
return _.all(attribs, function (attrib) {
return datum[attrib] === comparedValue[attrib];
});
});
};
sanitize = function sanitize(data) {
var allProblems = {},
tableNames = _.sortBy(_.keys(data.data), function (tableName) {
// We want to guarantee posts and tags go first
if (tableName === 'posts') {
return 1;
} else if (tableName === 'tags') {
return 2;
}
return 3;
});
_.each(tableNames, function (tableName) {
// Sanitize the table data for duplicates and valid uuid values
var sanitizedTableData = _.transform(data.data[tableName], function (memo, importValues) {
var uuidMissing = (!importValues.uuid && tables[tableName].uuid) ? true : false,
uuidMalformed = (importValues.uuid && !validator.isUUID(importValues.uuid)) ? true : false,
isDuplicate,
problemTag;
// Check for correct UUID and fix if neccessary
if (uuidMissing || uuidMalformed) {
importValues.uuid = uuid.v4();
}
// Custom sanitize for posts, tags and users
if (tableName === 'posts') {
// Check if any previously added posts have the same
// title and slug
isDuplicate = checkDuplicateAttributes(memo.data, importValues, ['title', 'slug']);
// If it's a duplicate add to the problems and continue on
if (isDuplicate) {
// TODO: Put the reason why it was a problem?
memo.problems.push(importValues);
return;
}
} else if (tableName === 'tags') {
// Check if any previously added posts have the same
// name and slug
isDuplicate = checkDuplicateAttributes(memo.data, importValues, ['name', 'slug']);
// If it's a duplicate add to the problems and continue on
if (isDuplicate) {
// TODO: Put the reason why it was a problem?
// Remember this tag so it can be updated later
importValues.duplicate = isDuplicate;
memo.problems.push(importValues);
return;
}
} else if (tableName === 'posts_tags') {
// Fix up removed tags associations
problemTag = _.find(allProblems.tags, function (tag) {
return tag.id === importValues.tag_id;
});
// Update the tag id to the original "duplicate" id
if (problemTag) {
importValues.tag_id = problemTag.duplicate.id;
}
}
memo.data.push(importValues);
}, {
data: [],
problems: []
});
// Store the table data to return
data.data[tableName] = sanitizedTableData.data;
// Keep track of all problems for all tables
if (!_.isEmpty(sanitizedTableData.problems)) {
allProblems[tableName] = sanitizedTableData.problems;
}
});
return data;
return {
data: data,
problems: allProblems
};
};
validate = function validate(data) {
@ -106,9 +185,12 @@ validate = function validate(data) {
};
module.exports = function (version, data) {
var importer;
var importer,
sanitizeResults;
data = sanitize(data);
sanitizeResults = sanitize(data);
data = sanitizeResults.data;
return validate(data).then(function () {
try {
@ -122,6 +204,8 @@ module.exports = function (version, data) {
}
return importer.importData(data);
}).then(function () {
return sanitizeResults;
}).catch(function (result) {
return handleErrors(result);
});

View file

@ -106,6 +106,70 @@ describe('Import', function () {
});
});
describe('Sanitizes', function () {
before(function () {
knex = config.database.knex;
});
beforeEach(testUtils.setup('roles', 'owner', 'settings'));
it('import results have data and problems', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-003').then(function (exported) {
exportData = exported;
return importer('003', exportData);
}).then(function (importResult) {
should.exist(importResult);
should.exist(importResult.data);
should.exist(importResult.problems);
done();
}).catch(done);
});
it('removes duplicate posts', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-003-duplicate-posts').then(function (exported) {
exportData = exported;
return importer('003', exportData);
}).then(function (importResult) {
should.exist(importResult.data.data.posts);
importResult.data.data.posts.length.should.equal(1);
importResult.problems.posts.length.should.equal(1);
done();
}).catch(done);
});
it('removes duplicate tags and updates associations', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-003-duplicate-tags').then(function (exported) {
exportData = exported;
return importer('003', exportData);
}).then(function (importResult) {
should.exist(importResult.data.data.tags);
should.exist(importResult.data.data.posts_tags);
importResult.data.data.tags.length.should.equal(1);
// Check we imported all posts_tags associations
importResult.data.data.posts_tags.length.should.equal(2);
// Check the post_tag.tag_id was updated when we removed duplicate tag
_.all(importResult.data.data.posts_tags, function (postTag) {
return postTag.tag_id !== 2;
});
importResult.problems.tags.length.should.equal(1);
done();
}).catch(done);
});
});
describe('000', function () {
before(function () {
knex = config.database.knex;

View file

@ -0,0 +1,391 @@
{
"meta": {
"exported_on": 1388318311015,
"version": "003"
},
"data": {
"posts": [
{
"id": 1,
"uuid": "8492fbba-1102-4b53-8e3e-abe207952f0c",
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice.",
"html": "<p>You're live! Nice.</p>",
"image": null,
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310782,
"created_by": 1,
"updated_at": 1388318310782,
"updated_by": 1,
"published_at": 1388318310783,
"published_by": 1
},
{
"id": 2,
"uuid": "8492fabb-1102-4b53-8e3e-abe207952f0c",
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice.",
"html": "<p>You're live! Nice.</p>",
"image": null,
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310782,
"created_by": 1,
"updated_at": 1388318310782,
"updated_by": 1,
"published_at": 1388318310783,
"published_by": 1
}
],
"users": [
{
"id": 1,
"uuid": "e5188224-4742-4c32-a2d6-e9c5c5d4c123",
"name": "Joe Bloggs",
"slug": "joe-bloggs",
"password": "$2a$10$.pZeeBE0gHXd0PTnbT/ph.GEKgd0Wd3q2pWna3ynTGBkPKnGIKABC",
"email": "jbloggs@example.com",
"image": null,
"cover": null,
"bio": "A blogger",
"website": null,
"location": null,
"accessibility": null,
"status": "active",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"last_login": null,
"created_at": 1388319501897,
"created_by": 1,
"updated_at": null,
"updated_by": null
}
],
"roles": [
{
"id": 1,
"uuid": "d2ea9c7f-7e6b-4cae-b009-35c298206852",
"name": "Administrator",
"description": "Administrators",
"created_at": 1388318310794,
"created_by": 1,
"updated_at": 1388318310794,
"updated_by": 1
},
{
"id": 2,
"uuid": "b0d7d6b0-5b88-45b5-b0e5-a487741b843d",
"name": "Editor",
"description": "Editors",
"created_at": 1388318310796,
"created_by": 1,
"updated_at": 1388318310796,
"updated_by": 1
},
{
"id": 3,
"uuid": "9f72e817-5490-4ccf-bc78-c557dc9613ca",
"name": "Author",
"description": "Authors",
"created_at": 1388318310799,
"created_by": 1,
"updated_at": 1388318310799,
"updated_by": 1
}
],
"roles_users": [
{
"id": 1,
"role_id": 1,
"user_id": 1
}
],
"permissions": [
{
"id": 1,
"uuid": "bdfbd261-e0fb-4c8e-abab-aece7a9e8e34",
"name": "Edit posts",
"object_type": "post",
"action_type": "edit",
"object_id": null,
"created_at": 1388318310803,
"created_by": 1,
"updated_at": 1388318310803,
"updated_by": 1
},
{
"id": 2,
"uuid": "580d31c4-e3db-40f3-969d-9a1caea9d1bb",
"name": "Remove posts",
"object_type": "post",
"action_type": "remove",
"object_id": null,
"created_at": 1388318310814,
"created_by": 1,
"updated_at": 1388318310814,
"updated_by": 1
},
{
"id": 3,
"uuid": "c1f8b024-e383-494a-835d-6fb673f143db",
"name": "Create posts",
"object_type": "post",
"action_type": "create",
"object_id": null,
"created_at": 1388318310818,
"created_by": 1,
"updated_at": 1388318310818,
"updated_by": 1
}
],
"permissions_users": [],
"permissions_roles": [
{
"id": 1,
"role_id": 1,
"permission_id": 1
},
{
"id": 2,
"role_id": 1,
"permission_id": 2
},
{
"id": 3,
"role_id": 1,
"permission_id": 3
}
],
"settings": [
{
"id": 1,
"uuid": "f90aa810-4fa2-49fe-a39b-7c0d2ebb473e",
"key": "databaseVersion",
"value": "001",
"type": "core",
"created_at": 1388318310829,
"created_by": 1,
"updated_at": 1388318310829,
"updated_by": 1
},
{
"id": 2,
"uuid": "95ce1c53-69b0-4f5f-be91-d3aeb39046b5",
"key": "dbHash",
"value": null,
"type": "core",
"created_at": 1388318310829,
"created_by": 1,
"updated_at": 1388318310829,
"updated_by": 1
},
{
"id": 3,
"uuid": "c356fbde-0bc5-4fe1-9309-2510291aa34d",
"key": "title",
"value": "Ghost",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 4,
"uuid": "858dc11f-8f9e-4011-99ee-d94c48d5a2ce",
"key": "description",
"value": "Just a blogging platform.",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 5,
"uuid": "37ca5ae7-bca6-4dd5-8021-4ef6c6dcb097",
"key": "email",
"value": "josephinebloggs@example.com",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 6,
"uuid": "1672d62c-fab7-4f22-b333-8cf760189f67",
"key": "logo",
"value": "",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 7,
"uuid": "cd8b0456-578b-467a-857e-551bad17a14d",
"key": "cover",
"value": "",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 8,
"uuid": "c4a074a4-05c7-49f7-83eb-068302c15d82",
"key": "defaultLang",
"value": "en_US",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 9,
"uuid": "21f2f5da-9bee-4dae-b3b7-b8d7baf8be33",
"key": "postsPerPage",
"value": "6",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 10,
"uuid": "2d21b736-f85a-4119-a0e3-5fc898b1bf47",
"key": "forceI18n",
"value": "true",
"type": "blog",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 11,
"uuid": "5c5b91b8-6062-4104-b855-9e121f72b0f0",
"key": "permalinks",
"value": "/:slug/",
"type": "blog",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 12,
"uuid": "795cb328-3e38-4906-81a8-fcdff19d914f",
"key": "activeTheme",
"value": "notcasper",
"type": "theme",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 13,
"uuid": "f3afce35-5166-453e-86c3-50dfff74dca7",
"key": "activeApps",
"value": "[]",
"type": "plugin",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 14,
"uuid": "2ea560a3-2304-449d-a62b-f7b622987510",
"key": "installedApps",
"value": "[]",
"type": "plugin",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
}
],
"tags": [
{
"id": 1,
"uuid": "a950117a-9735-4584-931d-25a28015a80d",
"name": "Getting Started",
"slug": "getting-started",
"description": null,
"parent_id": null,
"meta_title": null,
"meta_description": null,
"created_at": 1388318310790,
"created_by": 1,
"updated_at": 1388318310790,
"updated_by": 1
}
],
"posts_tags": [
{
"id": 1,
"post_id": 1,
"tag_id": 1
}
],
"apps": [
{
"id": 1,
"uuid": "4d7557f0-0949-4946-9fe8-ec030e0727f0",
"name": "Kudos",
"slug": "kudos",
"version": "0.0.1",
"status": "installed",
"created_at": 1388318312790,
"created_by": 1,
"updated_at": 1388318312790,
"updated_by": 1
}
],
"app_settings": [
{
"id": 1,
"uuid": "790e4551-b9cc-4954-8f5d-b6e651bc7342",
"key": "position",
"value": "bottom",
"app_id": 1,
"created_at": 1388318312790,
"created_by": 1,
"updated_at": 1388318312790,
"updated_by": 1
},
{
"id": 2,
"uuid": "29682b66-cdeb-4773-9821-bcf40ea93b58",
"key": "size",
"value": "60",
"app_id": 1,
"created_at": 1388318312790,
"created_by": 1,
"updated_at": 1388318312790,
"updated_by": 1
}
]
}
}

View file

@ -0,0 +1,388 @@
{
"meta": {
"exported_on": 1388318311015,
"version": "003"
},
"data": {
"posts": [
{
"id": 1,
"uuid": "8492fbba-1102-4b53-8e3e-abe207952f0c",
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice.",
"html": "<p>You're live! Nice.</p>",
"image": null,
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310782,
"created_by": 1,
"updated_at": 1388318310782,
"updated_by": 1,
"published_at": 1388318310783,
"published_by": 1
}
],
"users": [
{
"id": 1,
"uuid": "e5188224-4742-4c32-a2d6-e9c5c5d4c123",
"name": "Joe Bloggs",
"slug": "joe-bloggs",
"password": "$2a$10$.pZeeBE0gHXd0PTnbT/ph.GEKgd0Wd3q2pWna3ynTGBkPKnGIKABC",
"email": "jbloggs@example.com",
"image": null,
"cover": null,
"bio": "A blogger",
"website": null,
"location": null,
"accessibility": null,
"status": "active",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"last_login": null,
"created_at": 1388319501897,
"created_by": 1,
"updated_at": null,
"updated_by": null
}
],
"roles": [
{
"id": 1,
"uuid": "d2ea9c7f-7e6b-4cae-b009-35c298206852",
"name": "Administrator",
"description": "Administrators",
"created_at": 1388318310794,
"created_by": 1,
"updated_at": 1388318310794,
"updated_by": 1
},
{
"id": 2,
"uuid": "b0d7d6b0-5b88-45b5-b0e5-a487741b843d",
"name": "Editor",
"description": "Editors",
"created_at": 1388318310796,
"created_by": 1,
"updated_at": 1388318310796,
"updated_by": 1
},
{
"id": 3,
"uuid": "9f72e817-5490-4ccf-bc78-c557dc9613ca",
"name": "Author",
"description": "Authors",
"created_at": 1388318310799,
"created_by": 1,
"updated_at": 1388318310799,
"updated_by": 1
}
],
"roles_users": [
{
"id": 1,
"role_id": 1,
"user_id": 1
}
],
"permissions": [
{
"id": 1,
"uuid": "bdfbd261-e0fb-4c8e-abab-aece7a9e8e34",
"name": "Edit posts",
"object_type": "post",
"action_type": "edit",
"object_id": null,
"created_at": 1388318310803,
"created_by": 1,
"updated_at": 1388318310803,
"updated_by": 1
},
{
"id": 2,
"uuid": "580d31c4-e3db-40f3-969d-9a1caea9d1bb",
"name": "Remove posts",
"object_type": "post",
"action_type": "remove",
"object_id": null,
"created_at": 1388318310814,
"created_by": 1,
"updated_at": 1388318310814,
"updated_by": 1
},
{
"id": 3,
"uuid": "c1f8b024-e383-494a-835d-6fb673f143db",
"name": "Create posts",
"object_type": "post",
"action_type": "create",
"object_id": null,
"created_at": 1388318310818,
"created_by": 1,
"updated_at": 1388318310818,
"updated_by": 1
}
],
"permissions_users": [],
"permissions_roles": [
{
"id": 1,
"role_id": 1,
"permission_id": 1
},
{
"id": 2,
"role_id": 1,
"permission_id": 2
},
{
"id": 3,
"role_id": 1,
"permission_id": 3
}
],
"settings": [
{
"id": 1,
"uuid": "f90aa810-4fa2-49fe-a39b-7c0d2ebb473e",
"key": "databaseVersion",
"value": "001",
"type": "core",
"created_at": 1388318310829,
"created_by": 1,
"updated_at": 1388318310829,
"updated_by": 1
},
{
"id": 2,
"uuid": "95ce1c53-69b0-4f5f-be91-d3aeb39046b5",
"key": "dbHash",
"value": null,
"type": "core",
"created_at": 1388318310829,
"created_by": 1,
"updated_at": 1388318310829,
"updated_by": 1
},
{
"id": 3,
"uuid": "c356fbde-0bc5-4fe1-9309-2510291aa34d",
"key": "title",
"value": "Ghost",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 4,
"uuid": "858dc11f-8f9e-4011-99ee-d94c48d5a2ce",
"key": "description",
"value": "Just a blogging platform.",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 5,
"uuid": "37ca5ae7-bca6-4dd5-8021-4ef6c6dcb097",
"key": "email",
"value": "josephinebloggs@example.com",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 6,
"uuid": "1672d62c-fab7-4f22-b333-8cf760189f67",
"key": "logo",
"value": "",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 7,
"uuid": "cd8b0456-578b-467a-857e-551bad17a14d",
"key": "cover",
"value": "",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 8,
"uuid": "c4a074a4-05c7-49f7-83eb-068302c15d82",
"key": "defaultLang",
"value": "en_US",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 9,
"uuid": "21f2f5da-9bee-4dae-b3b7-b8d7baf8be33",
"key": "postsPerPage",
"value": "6",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 10,
"uuid": "2d21b736-f85a-4119-a0e3-5fc898b1bf47",
"key": "forceI18n",
"value": "true",
"type": "blog",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 11,
"uuid": "5c5b91b8-6062-4104-b855-9e121f72b0f0",
"key": "permalinks",
"value": "/:slug/",
"type": "blog",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 12,
"uuid": "795cb328-3e38-4906-81a8-fcdff19d914f",
"key": "activeTheme",
"value": "notcasper",
"type": "theme",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 13,
"uuid": "f3afce35-5166-453e-86c3-50dfff74dca7",
"key": "activeApps",
"value": "[]",
"type": "plugin",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 14,
"uuid": "2ea560a3-2304-449d-a62b-f7b622987510",
"key": "installedApps",
"value": "[]",
"type": "plugin",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
}
],
"tags": [
{
"id": 1,
"uuid": "a950117a-9735-4584-931d-25a28015a80d",
"name": "Getting Started",
"slug": "getting-started",
"description": null,
"parent_id": null,
"meta_title": null,
"meta_description": null,
"created_at": 1388318310790,
"created_by": 1,
"updated_at": 1388318310790,
"updated_by": 1
},
{
"id": 2,
"uuid": "a950117b-9735-4584-931d-25a28015a80d",
"name": "Getting Started",
"slug": "getting-started",
"description": null,
"parent_id": null,
"meta_title": null,
"meta_description": null,
"created_at": 1388318310790,
"created_by": 1,
"updated_at": 1388318310790,
"updated_by": 1
}
],
"posts_tags": [
{
"id": 1,
"post_id": 1,
"tag_id": 1
},
{
"id": 2,
"post_id": 1,
"tag_id": 2
}
],
"apps": [
{
"id": 1,
"uuid": "4d7557f0-0949-4946-9fe8-ec030e0727f0",
"name": "Kudos",
"slug": "kudos",
"version": "0.0.1",
"status": "installed",
"created_at": 1388318312790,
"created_by": 1,
"updated_at": 1388318312790,
"updated_by": 1
}
],
"app_settings": [
{
"id": 1,
"uuid": "790e4551-b9cc-4954-8f5d-b6e651bc7342",
"key": "position",
"value": "bottom",
"app_id": 1,
"created_at": 1388318312790,
"created_by": 1,
"updated_at": 1388318312790,
"updated_by": 1
},
{
"id": 2,
"uuid": "29682b66-cdeb-4773-9821-bcf40ea93b58",
"key": "size",
"value": "60",
"app_id": 1,
"created_at": 1388318312790,
"created_by": 1,
"updated_at": 1388318312790,
"updated_by": 1
}
]
}
}