tqt/Ghost
2
1
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2023-12-13 21:00:40 +01:00
Code Issues 1 Releases Wiki Activity

Deduplicated batch insertion code

Browse source 
refs https://github.com/TryGhost/DevOps/issues/119

- this function can simply call the `import` function, which performs
  the same code as we had here
- this makes the code cleaner to read and understand
This commit is contained in:
Daniel Lockyer 2023-12-07 12:26:48 +01:00 committed by Daniel Lockyer
parent 76455cb64a
commit ff34a98b94
3 changed files with 34 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
ghost/data-generator/lib/importers/TableImporter.js
View file

@ -1,3 +1,5 @@
const debug = require('@tryghost/debug')('TableImporter');
class TableImporter {
/**
* @type {object|undefined} model Referenced model when generating data
@ -21,27 +23,29 @@ class TableImporter {
this.transaction = transaction;
}
async import(amount = this.defaultQuantity) {
const batchSize = 500;
let batch = [];
async #generateData(amount = this.defaultQuantity) {
let data = [];
for (let i = 0; i < amount; i++) {
const model = await this.generate();
if (model) {
batch.push(model);
} else {
// After first null assume that there is no more data
break;
}
if (batch.length === batchSize) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
batch = [];
data.push(model);
}
}
// Process final batch
if (batch.length > 0) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
return data;
}
async import(amount = this.defaultQuantity) {
const generateNow = Date.now();
const data = await this.#generateData(amount);
debug(`${this.name} generated ${data.length} records in ${Date.now() - generateNow}ms`);
if (data.length > 0) {
debug (`Importing ${data.length} records into ${this.name}`);
const now = Date.now();
await this.knex.batchInsert(this.name, data).transacting(this.transaction);
debug(`${this.name} imported ${data.length} records in ${Date.now() - now}ms`);
}
}
@ -50,8 +54,10 @@ class TableImporter {
* @param {Number|function} amount Number of records to import per model
*/
async importForEach(models = [], amount) {
const batchSize = 500;
let batch = [];
const data = [];
debug (`Generating data for ${models.length} models for ${this.name}`);
const now = Date.now();
for (const model of models) {
this.setReferencedModel(model);
@ -59,24 +65,19 @@ class TableImporter {
if (!Number.isInteger(currentAmount)) {
currentAmount = Math.floor(currentAmount) + ((Math.random() < currentAmount % 1) ? 1 : 0);
}
for (let i = 0; i < currentAmount; i++) {
const data = await this.generate();
if (data) {
batch.push(data);
} else {
// After first null assume that there is no more data for this model
break;
}
if (batch.length === batchSize) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
batch = [];
}
const generatedData = await this.#generateData(currentAmount);
if (generatedData.length > 0) {
data.push(...generatedData);
}
}
// Process final batch
if (batch.length > 0) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
debug(`${this.name} generated ${data.length} records in ${Date.now() - now}ms`);
if (data.length > 0) {
const now2 = Date.now();
await this.knex.batchInsert(this.name, data).transacting(this.transaction);
debug(`${this.name} imported ${data.length} records in ${Date.now() - now2}ms`);
}
}

1
ghost/data-generator/package.json
View file

@ -18,6 +18,7 @@
"lib"
],
"devDependencies": {
"@tryghost/debug": "0.1.27",
"c8": "8.0.1",
"knex": "2.4.2",
"mocha": "10.2.0",

2
yarn.lock
View file

@ -7623,7 +7623,7 @@
"@tryghost/root-utils" "^0.3.24"
debug "^4.3.1"
"@tryghost/debug@^0.1.13", "@tryghost/debug@^0.1.26", "@tryghost/debug@^0.1.27":
"@tryghost/debug@0.1.27", "@tryghost/debug@^0.1.13", "@tryghost/debug@^0.1.26", "@tryghost/debug@^0.1.27":
version "0.1.27"
resolved "https://registry.yarnpkg.com/@tryghost/debug/-/debug-0.1.27.tgz#e1e08e4b69b236de0996a710bac8bcb2d15552bc"
integrity sha512-pLFSm+1FLYKKnjVR5B8dXC5ArjU9AbbW3IBv1oAtupjFDf+rtN/ShMgaaLI8c4c9G/F6fAtvnO/yOJ2ea85W4A==