tqt
/
Ghost
mirror of https://github.com/TryGhost/Ghost.git
2
1
Fork 0
Code Issues 1 Releases Wiki Activity

Extracted members CSV related code into @tryghost/members-csv package 

no issue

- Moves out CSV parsing and serialization related code into separate package as a part of push to modularize Ghost repo.
- Next up is to remove `csv-parser` dependency from this new package
This commit is contained in:
Nazar Gargol 2020-06-19 19:55:39 +12:00
parent fabe06c5c5
commit 93e8ee83d9
9 changed files with 11 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/server/api/canary/utils/serializers/input/members.js
View File

@ -1,6 +1,6 @@
const _ = require('lodash');
const debug = require('ghost-ignition').debug('api:canary:utils:serializers:input:members');
const {parse} = require('./utils/members-import-csv');
const {parse} = require('@tryghost/members-csv');
function defaultRelations(frame) {
if (frame.options.withRelated) {

92
core/server/api/canary/utils/serializers/input/utils/members-import-csv.js
View File

@ -1,92 +0,0 @@
const Promise = require('bluebird');
const csvParser = require('csv-parser');
const _ = require('lodash');
const fs = require('fs-extra');
const readCSV = (options) => {
const columnsToExtract = options.columnsToExtract || [];
let results = [];
const rows = [];
return new Promise(function (resolve, reject) {
const readFile = fs.createReadStream(options.path);
readFile.on('err', function (err) {
reject(err);
})
.pipe(csvParser())
.on('data', function (row) {
rows.push(row);
})
.on('end', function () {
// If CSV is single column - return all values including header
const headers = _.keys(rows[0]);
let result = {};
const columnMap = {};
if (columnsToExtract.length === 1 && headers.length === 1) {
results = _.map(rows, function (value) {
result = {};
result[columnsToExtract[0].name] = value[headers[0]];
return result;
});
} else {
// If there are multiple columns in csv file
// try to match headers using lookup value
_.map(columnsToExtract, function findMatches(column) {
_.each(headers, function checkheader(header) {
if (column.lookup.test(header)) {
columnMap[column.name] = header;
}
});
});
results = _.map(rows, function evaluateRow(row) {
const result = {};
_.each(columnMap, function returnMatches(value, key) {
result[key] = row[value];
});
return result;
});
}
resolve(results);
});
});
};
const parse = async (filePath) => {
const columnsToExtract = [{
name: 'email',
lookup: /^email/i
}, {
name: 'name',
lookup: /name/i
}, {
name: 'note',
lookup: /note/i
}, {
name: 'subscribed_to_emails',
lookup: /subscribed_to_emails/i
}, {
name: 'stripe_customer_id',
lookup: /stripe_customer_id/i
}, {
name: 'complimentary_plan',
lookup: /complimentary_plan/i
}, {
name: 'labels',
lookup: /labels/i
}, {
name: 'created_at',
lookup: /created_at/i
}];
return await readCSV({
path: filePath,
columnsToExtract: columnsToExtract
});
};
module.exports.parse = parse;
module.exports.readCSV = readCSV;

2
core/server/api/canary/utils/serializers/output/members.js
View File

@ -2,7 +2,7 @@ const {i18n} = require('../../../../../lib/common');
const errors = require('@tryghost/errors');
const debug = require('ghost-ignition').debug('api:canary:utils:serializers:output:members');
const mapper = require('./utils/mapper');
const {unparse} = require('./utils/members-csv');
const {unparse} = require('@tryghost/members-csv');
module.exports = {
hasActiveStripeSubscriptions(data, apiConfig, frame) {

33
core/server/api/canary/utils/serializers/output/utils/members-csv.js
View File

@ -1,33 +0,0 @@
const _ = require('lodash');
const papaparse = require('papaparse');
const unparse = (members) => {
const mappedMembers = members.map((member) => {
let stripeCustomerId;
if (member.stripe) {
stripeCustomerId = _.get(member, 'stripe.subscriptions[0].customer.id');
}
let labels = [];
if (member.labels) {
labels = `${member.labels.map(l => l.name).join(',')}`;
}
return {
id: member.id,
email: member.email,
name: member.name,
note: member.note,
subscribed_to_emails: member.subscribed,
complimentary_plan: member.comped,
stripe_customer_id: stripeCustomerId,
created_at: member.created_at,
deleted_at: member.deleted_at,
labels: labels
};
});
return papaparse.unparse(mappedMembers);
};
module.exports.unparse = unparse;

3
package.json
View File

@ -53,6 +53,7 @@
"@tryghost/kg-mobiledoc-html-renderer": "3.0.1",
"@tryghost/magic-link": "0.4.8",
"@tryghost/members-api": "0.23.0",
"@tryghost/members-csv": "0.1.2",
"@tryghost/members-ssr": "0.8.1",
"@tryghost/mw-session-from-token": "0.1.4",
"@tryghost/session-service": "0.1.4",
@ -76,7 +77,6 @@
"connect-slashes": "1.4.0",
"cookie-session": "1.4.0",
"cors": "2.8.5",
"csv-parser": "2.3.3",
"downsize": "0.0.8",
"express": "4.17.1",
"express-brute": "1.0.1",
@ -121,7 +121,6 @@
"node-jose": "1.1.4",
"nodemailer": "0.7.1",
"oembed-parser": "1.3.7",
"papaparse": "5.2.0",
"path-match": "1.2.4",
"probe-image-size": "5.0.0",
"rss": "1.2.2",

53
test/unit/api/canary/utils/serializers/input/utils/members-import-csv_spec.js
View File

@ -1,53 +0,0 @@
const should = require('should');
const path = require('path');
const fsLib = require('../../../../../../../../core/server/api/canary/utils/serializers/input/utils/members-import-csv');
const csvPath = path.join(__dirname, '../../../../../../../utils/fixtures/csv/');
describe('members-import-csv: read csv', function () {
it('read csv: one column', function (done) {
fsLib.readCSV({
path: csvPath + 'single-column-with-header.csv',
columnsToExtract: [{name: 'email', lookup: /email/i}]
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('jbloggs@example.com');
result[1].email.should.eql('test@example.com');
done();
}).catch(done);
});
it('read csv: two columns, 1 filter', function (done) {
fsLib.readCSV({
path: csvPath + 'two-columns-with-header.csv',
columnsToExtract: [{name: 'email', lookup: /email/i}]
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('jbloggs@example.com');
result[1].email.should.eql('test@example.com');
should.not.exist(result[0].id);
done();
}).catch(done);
});
it('read csv: two columns, 2 filters', function (done) {
fsLib.readCSV({
path: csvPath + 'two-columns-obscure-header.csv',
columnsToExtract: [
{name: 'email', lookup: /email/i},
{name: 'id', lookup: /id/i}
]
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('jbloggs@example.com');
result[0].id.should.eql('1');
result[1].email.should.eql('test@example.com');
result[1].id.should.eql('2');
done();
}).catch(done);
});
});

3
test/utils/fixtures/csv/two-columns-obscure-header.csv
View File

@ -1,3 +0,0 @@
id,Email Address
1,"jbloggs@example.com"
2,test@example.com
1 id Email Address
2 1 jbloggs@example.com
3 2 test@example.com

3
test/utils/fixtures/csv/two-columns-with-header.csv
View File

@ -1,3 +0,0 @@
id,email
1,"jbloggs@example.com"
1,test@example.com
1 id email
2 1 jbloggs@example.com
3 1 test@example.com

8
yarn.lock
View File

@ -513,6 +513,14 @@
node-jose "^1.1.3"
stripe "^7.4.0"
"@tryghost/members-csv@0.1.2":
version "0.1.2"
resolved "https://registry.yarnpkg.com/@tryghost/members-csv/-/members-csv-0.1.2.tgz#c543125ed4cf3e02a2fcb3d98d365dd2e1796047"
integrity sha512-KJgtgbDQbgRS4K3OmulqZ64jIvm4+66f/RNhK0vwkf3HGgW5JEv5PVJJbVCu5IbwubwCYUjz/0K3oyl+BqIh0w==
dependencies:
csv-parser "2.3.3"
papaparse "5.2.0"
"@tryghost/members-ssr@0.8.1":
version "0.8.1"
resolved "https://registry.yarnpkg.com/@tryghost/members-ssr/-/members-ssr-0.8.1.tgz#061cf38b902da9309e0a499fed80ea5d013e3f8c"