From 93e8ee83d9f23f3dd311df645b613f607cfd5c9d Mon Sep 17 00:00:00 2001 From: Nazar Gargol Date: Fri, 19 Jun 2020 19:55:39 +1200 Subject: [PATCH] Extracted members CSV related code into @tryghost/members-csv package no issue - Moves out CSV parsing and serialization related code into separate package as a part of push to modularize Ghost repo. - Next up is to remove `csv-parser` dependency from this new package --- .../canary/utils/serializers/input/members.js | 2 +- .../input/utils/members-import-csv.js | 92 ------------------- .../utils/serializers/output/members.js | 2 +- .../serializers/output/utils/members-csv.js | 33 ------- package.json | 3 +- .../input/utils/members-import-csv_spec.js | 53 ----------- .../csv/two-columns-obscure-header.csv | 3 - .../fixtures/csv/two-columns-with-header.csv | 3 - yarn.lock | 8 ++ 9 files changed, 11 insertions(+), 188 deletions(-) delete mode 100644 core/server/api/canary/utils/serializers/input/utils/members-import-csv.js delete mode 100644 core/server/api/canary/utils/serializers/output/utils/members-csv.js delete mode 100644 test/unit/api/canary/utils/serializers/input/utils/members-import-csv_spec.js delete mode 100644 test/utils/fixtures/csv/two-columns-obscure-header.csv delete mode 100644 test/utils/fixtures/csv/two-columns-with-header.csv diff --git a/core/server/api/canary/utils/serializers/input/members.js b/core/server/api/canary/utils/serializers/input/members.js index bf64622a66..e71b0c9c78 100644 --- a/core/server/api/canary/utils/serializers/input/members.js +++ b/core/server/api/canary/utils/serializers/input/members.js @@ -1,6 +1,6 @@ const _ = require('lodash'); const debug = require('ghost-ignition').debug('api:canary:utils:serializers:input:members'); -const {parse} = require('./utils/members-import-csv'); +const {parse} = require('@tryghost/members-csv'); function defaultRelations(frame) { if (frame.options.withRelated) { diff --git a/core/server/api/canary/utils/serializers/input/utils/members-import-csv.js b/core/server/api/canary/utils/serializers/input/utils/members-import-csv.js deleted file mode 100644 index 202705a866..0000000000 --- a/core/server/api/canary/utils/serializers/input/utils/members-import-csv.js +++ /dev/null @@ -1,92 +0,0 @@ -const Promise = require('bluebird'); -const csvParser = require('csv-parser'); -const _ = require('lodash'); -const fs = require('fs-extra'); - -const readCSV = (options) => { - const columnsToExtract = options.columnsToExtract || []; - let results = []; - const rows = []; - - return new Promise(function (resolve, reject) { - const readFile = fs.createReadStream(options.path); - - readFile.on('err', function (err) { - reject(err); - }) - .pipe(csvParser()) - .on('data', function (row) { - rows.push(row); - }) - .on('end', function () { - // If CSV is single column - return all values including header - const headers = _.keys(rows[0]); - - let result = {}; - const columnMap = {}; - if (columnsToExtract.length === 1 && headers.length === 1) { - results = _.map(rows, function (value) { - result = {}; - result[columnsToExtract[0].name] = value[headers[0]]; - return result; - }); - } else { - // If there are multiple columns in csv file - // try to match headers using lookup value - - _.map(columnsToExtract, function findMatches(column) { - _.each(headers, function checkheader(header) { - if (column.lookup.test(header)) { - columnMap[column.name] = header; - } - }); - }); - - results = _.map(rows, function evaluateRow(row) { - const result = {}; - _.each(columnMap, function returnMatches(value, key) { - result[key] = row[value]; - }); - return result; - }); - } - resolve(results); - }); - }); -}; - -const parse = async (filePath) => { - const columnsToExtract = [{ - name: 'email', - lookup: /^email/i - }, { - name: 'name', - lookup: /name/i - }, { - name: 'note', - lookup: /note/i - }, { - name: 'subscribed_to_emails', - lookup: /subscribed_to_emails/i - }, { - name: 'stripe_customer_id', - lookup: /stripe_customer_id/i - }, { - name: 'complimentary_plan', - lookup: /complimentary_plan/i - }, { - name: 'labels', - lookup: /labels/i - }, { - name: 'created_at', - lookup: /created_at/i - }]; - - return await readCSV({ - path: filePath, - columnsToExtract: columnsToExtract - }); -}; - -module.exports.parse = parse; -module.exports.readCSV = readCSV; diff --git a/core/server/api/canary/utils/serializers/output/members.js b/core/server/api/canary/utils/serializers/output/members.js index 60e00bb684..8fb4750e18 100644 --- a/core/server/api/canary/utils/serializers/output/members.js +++ b/core/server/api/canary/utils/serializers/output/members.js @@ -2,7 +2,7 @@ const {i18n} = require('../../../../../lib/common'); const errors = require('@tryghost/errors'); const debug = require('ghost-ignition').debug('api:canary:utils:serializers:output:members'); const mapper = require('./utils/mapper'); -const {unparse} = require('./utils/members-csv'); +const {unparse} = require('@tryghost/members-csv'); module.exports = { hasActiveStripeSubscriptions(data, apiConfig, frame) { diff --git a/core/server/api/canary/utils/serializers/output/utils/members-csv.js b/core/server/api/canary/utils/serializers/output/utils/members-csv.js deleted file mode 100644 index 72a679e103..0000000000 --- a/core/server/api/canary/utils/serializers/output/utils/members-csv.js +++ /dev/null @@ -1,33 +0,0 @@ -const _ = require('lodash'); -const papaparse = require('papaparse'); - -const unparse = (members) => { - const mappedMembers = members.map((member) => { - let stripeCustomerId; - - if (member.stripe) { - stripeCustomerId = _.get(member, 'stripe.subscriptions[0].customer.id'); - } - let labels = []; - if (member.labels) { - labels = `${member.labels.map(l => l.name).join(',')}`; - } - - return { - id: member.id, - email: member.email, - name: member.name, - note: member.note, - subscribed_to_emails: member.subscribed, - complimentary_plan: member.comped, - stripe_customer_id: stripeCustomerId, - created_at: member.created_at, - deleted_at: member.deleted_at, - labels: labels - }; - }); - - return papaparse.unparse(mappedMembers); -}; - -module.exports.unparse = unparse; diff --git a/package.json b/package.json index 1e1240ad8e..be821c38e5 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "@tryghost/kg-mobiledoc-html-renderer": "3.0.1", "@tryghost/magic-link": "0.4.8", "@tryghost/members-api": "0.23.0", + "@tryghost/members-csv": "0.1.2", "@tryghost/members-ssr": "0.8.1", "@tryghost/mw-session-from-token": "0.1.4", "@tryghost/session-service": "0.1.4", @@ -76,7 +77,6 @@ "connect-slashes": "1.4.0", "cookie-session": "1.4.0", "cors": "2.8.5", - "csv-parser": "2.3.3", "downsize": "0.0.8", "express": "4.17.1", "express-brute": "1.0.1", @@ -121,7 +121,6 @@ "node-jose": "1.1.4", "nodemailer": "0.7.1", "oembed-parser": "1.3.7", - "papaparse": "5.2.0", "path-match": "1.2.4", "probe-image-size": "5.0.0", "rss": "1.2.2", diff --git a/test/unit/api/canary/utils/serializers/input/utils/members-import-csv_spec.js b/test/unit/api/canary/utils/serializers/input/utils/members-import-csv_spec.js deleted file mode 100644 index 19bbc184c8..0000000000 --- a/test/unit/api/canary/utils/serializers/input/utils/members-import-csv_spec.js +++ /dev/null @@ -1,53 +0,0 @@ -const should = require('should'); -const path = require('path'); -const fsLib = require('../../../../../../../../core/server/api/canary/utils/serializers/input/utils/members-import-csv'); - -const csvPath = path.join(__dirname, '../../../../../../../utils/fixtures/csv/'); - -describe('members-import-csv: read csv', function () { - it('read csv: one column', function (done) { - fsLib.readCSV({ - path: csvPath + 'single-column-with-header.csv', - columnsToExtract: [{name: 'email', lookup: /email/i}] - }).then(function (result) { - should.exist(result); - result.length.should.eql(2); - result[0].email.should.eql('jbloggs@example.com'); - result[1].email.should.eql('test@example.com'); - done(); - }).catch(done); - }); - - it('read csv: two columns, 1 filter', function (done) { - fsLib.readCSV({ - path: csvPath + 'two-columns-with-header.csv', - columnsToExtract: [{name: 'email', lookup: /email/i}] - }).then(function (result) { - should.exist(result); - result.length.should.eql(2); - result[0].email.should.eql('jbloggs@example.com'); - result[1].email.should.eql('test@example.com'); - should.not.exist(result[0].id); - - done(); - }).catch(done); - }); - - it('read csv: two columns, 2 filters', function (done) { - fsLib.readCSV({ - path: csvPath + 'two-columns-obscure-header.csv', - columnsToExtract: [ - {name: 'email', lookup: /email/i}, - {name: 'id', lookup: /id/i} - ] - }).then(function (result) { - should.exist(result); - result.length.should.eql(2); - result[0].email.should.eql('jbloggs@example.com'); - result[0].id.should.eql('1'); - result[1].email.should.eql('test@example.com'); - result[1].id.should.eql('2'); - done(); - }).catch(done); - }); -}); diff --git a/test/utils/fixtures/csv/two-columns-obscure-header.csv b/test/utils/fixtures/csv/two-columns-obscure-header.csv deleted file mode 100644 index 3c8fb4c34e..0000000000 --- a/test/utils/fixtures/csv/two-columns-obscure-header.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,Email Address -1,"jbloggs@example.com" -2,test@example.com diff --git a/test/utils/fixtures/csv/two-columns-with-header.csv b/test/utils/fixtures/csv/two-columns-with-header.csv deleted file mode 100644 index 37db487e4d..0000000000 --- a/test/utils/fixtures/csv/two-columns-with-header.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,email -1,"jbloggs@example.com" -1,test@example.com diff --git a/yarn.lock b/yarn.lock index f18f1b7604..1d6a0d3d2f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -513,6 +513,14 @@ node-jose "^1.1.3" stripe "^7.4.0" +"@tryghost/members-csv@0.1.2": + version "0.1.2" + resolved "https://registry.yarnpkg.com/@tryghost/members-csv/-/members-csv-0.1.2.tgz#c543125ed4cf3e02a2fcb3d98d365dd2e1796047" + integrity sha512-KJgtgbDQbgRS4K3OmulqZ64jIvm4+66f/RNhK0vwkf3HGgW5JEv5PVJJbVCu5IbwubwCYUjz/0K3oyl+BqIh0w== + dependencies: + csv-parser "2.3.3" + papaparse "5.2.0" + "@tryghost/members-ssr@0.8.1": version "0.8.1" resolved "https://registry.yarnpkg.com/@tryghost/members-ssr/-/members-ssr-0.8.1.tgz#061cf38b902da9309e0a499fed80ea5d013e3f8c"