Added email card and replacement handling to member emails

no issue

- adjusted mega's post serializer to get full email contents
  - fetch `mobiledoc` from the API rather than the pre-rendered `html` and `plaintext`
  - re-generate `html` using the mobiledoc renderer with an "email" target so that the email-only card content is included
  - re-generate `plaintext` from the newly generated email html

- added replacement handling to mega's `getEmailData` function
  - find all of our `%%{replacement "fallback"}%%` instances in the html template and push them into a replacements array with the respective property on the member instance and desired fallback
  - transform the replacement for Mailgun compatibility. Mailgun uses `%recipient.variable_name%` for its template variables so we need to replace our custom replacement string with the compatible version. Our replacements system allows for the same replacement (`{subscriber_name}`) to be used multiple times and have different fallbacks, Mailgun doesn't support fallbacks so for each replacement we also need an indexed `variable_name` part so that we can put our fallbacks in the correct place
  - perform the same Mailgun template transformation for the plaintext version except we re-use the replacements array to avoid bloating the API request to Mailgun with duplicate template variables for every recipient
  - swapped `reduce` for a plain loop for easier readability

core/server/services/mega/mega.js

@@ -8,19 +8,76 @@ const models = require('../../models');
 const postEmailSerializer = require('./post-email-serializer');
 const config = require('../../config');
 
-const getEmailData = async (postModel, recipients = []) => {
+const getEmailData = async (postModel, members = []) => {
     const emailTmpl = await postEmailSerializer.serialize(postModel);
     emailTmpl.from = membersService.config.getEmailFromAddress();
 
-    const emails = recipients.map(recipient => recipient.email);
-    const emailData = recipients.reduce((emailData, recipient) => {
-        return Object.assign({
-            [recipient.email]: {
-                unique_id: recipient.uuid,
-                unsubscribe_url: postEmailSerializer.createUnsubscribeUrl(recipient.uuid)
+    const EMAIL_REPLACEMENT_REGEX = /%%(\{.*?\})%%/g;
+    // the " is necessary here because `juice` will convert "->" for email compatibility
+    const REPLACEMENT_STRING_REGEX = /\{(?<memberProp>\w*?)(?:,? *(?:"|&quot;)(?<fallback>.*?)(?:"|&quot;))?\}/;
+    const ALLOWED_REPLACEMENTS = ['subscriber_firstname'];
+
+    // extract replacements with fallbacks. We have to handle replacements here because
+    // it's the only place we have access to both member data and specified fallbacks
+    const replacements = [];
+    emailTmpl.html = emailTmpl.html.replace(EMAIL_REPLACEMENT_REGEX, (replacementMatch, replacementStr) => {
+        const match = replacementStr.match(REPLACEMENT_STRING_REGEX);
+
+        if (match) {
+            const {memberProp, fallback} = match.groups;
+
+            if (ALLOWED_REPLACEMENTS.includes(memberProp)) {
+                const varName = `replacement_${replacements.length}`;
+
+                replacements.push({
+                    varName,
+                    memberProp,
+                    fallback
+                });
+                return `%recipient.${varName}%`;
             }
-        }, emailData);
-    }, {});
+        }
+
+        // output the user-entered replacement string for unknown or invalid replacements
+        // so that it's obvious there's an error in test emails
+        return replacementStr;
+    });
+
+    // plaintext will have the same replacements so no need to add them to the list and
+    // bloat the template variables object but we still need replacements for mailgun template syntax
+    let count = 0;
+    emailTmpl.plaintext = emailTmpl.plaintext.replace(EMAIL_REPLACEMENT_REGEX, (match, replacementStr) => {
+        const {groups: {memberProp}} = replacementStr.match(REPLACEMENT_STRING_REGEX);
+        if (ALLOWED_REPLACEMENTS.includes(memberProp)) {
+            const varName = `replacement_${count}`;
+            count++;
+            return `%recipient.${varName}`;
+        }
+        return replacementStr;
+    });
+
+    const emails = [];
+    const emailData = {};
+    members.forEach((member) => {
+        emails.push(member.email);
+
+        // firstname is a computed property only used here for now
+        // TODO: move into model computed property or output serializer?
+        member.firstname = (member.name || '').split(' ')[0];
+
+        // add static data to mailgun template variables
+        const data = {
+            unique_id: member.uuid,
+            unsubscribe_url: postEmailSerializer.createUnsubscribeUrl(member.uuid)
+        };
+
+        // add replacement data/requested fallback to mailgun template variables
+        replacements.forEach(({varName, memberProp, fallback}) => {
+            data[varName] = member[memberProp] || fallback || '';
+        });
+
+        emailData[member.email] = data;
+    });
 
     return {emailTmpl, emails, emailData};
 };
@@ -233,7 +290,9 @@ async function pendingEmailHandler(emailModel, options) {
 }
 
 const statusChangedHandler = (emailModel, options) => {
-    const emailRetried = emailModel.wasChanged() && (emailModel.get('status') === 'pending') && (emailModel.previous('status') === 'failed');
+    const emailRetried = emailModel.wasChanged()
+        && emailModel.get('status') === 'pending'
+        && emailModel.previous('status') === 'failed';
 
     if (emailRetried) {
         pendingEmailHandler(emailModel, options);

core/server/services/mega/post-email-serializer.js

@@ -6,6 +6,8 @@ const moment = require('moment');
 const cheerio = require('cheerio');
 const api = require('../../api');
 const {URL} = require('url');
+const mobiledocLib = require('../../lib/mobiledoc');
+const htmlToText = require('html-to-text');
 
 const getSite = () => {
     const publicSettings = settingsCache.getPublic();
@@ -39,7 +41,8 @@ const createUnsubscribeUrl = (uuid) => {
 // NOTE: serialization is needed to make sure we are using current API and do post transformations
 //       such as image URL transformation from relative to absolute
 const serializePostModel = async (model) => {
-    const frame = {options: {context: {user: true}, formats: 'html, plaintext'}};
+    // fetch mobiledoc rather than html and plaintext so we can render email-specific contents
+    const frame = {options: {context: {user: true}, formats: 'mobiledoc'}};
     const apiVersion = model.get('api_version') || 'v3';
     const docName = 'posts';
 
@@ -53,18 +56,32 @@ const serializePostModel = async (model) => {
 
 const serialize = async (postModel, options = {isBrowserPreview: false}) => {
     const post = await serializePostModel(postModel);
+
     post.published_at = post.published_at ? moment(post.published_at).format('DD MMM YYYY') : moment().format('DD MMM YYYY');
     post.authors = post.authors && post.authors.map(author => author.name).join(',');
-    post.html = post.html || '';
     if (post.posts_meta) {
         post.email_subject = post.posts_meta.email_subject;
     }
+    post.html = mobiledocLib.mobiledocHtmlRenderer.render(JSON.parse(post.mobiledoc), {target: 'email'});
+    // same options as used in Post model for generating plaintext but without `wordwrap: 80`
+    // to avoid replacement strings being split across lines and for mail clients to handle
+    // word wrapping based on user preferences
+    post.plaintext = htmlToText.fromString(post.html, {
+        ignoreImage: true,
+        hideLinkHrefIfSameAsText: true,
+        preserveNewlines: true,
+        returnDomByDefault: true,
+        uppercaseHeadings: false
+    });
+
     let htmlTemplate = template({post, site: getSite()});
     if (options.isBrowserPreview) {
         const previewUnsubscribeUrl = createUnsubscribeUrl();
         htmlTemplate = htmlTemplate.replace('%recipient.unsubscribe_url%', previewUnsubscribeUrl);
     }
+
     let juicedHtml = juice(htmlTemplate);
+
     // Force all links to open in new tab
     let _cheerio = cheerio.load(juicedHtml);
     _cheerio('a').attr('target','_blank');