turned SQLiteContactSource into a better example sync source

- simplified the database schema by removing unused tables
- added more per-contact properties missing in the original Apple schema
- added comments
- implemented storing of properties with 1:1 mapping to columns in database
- simplified the test cases used for sqlite

Client::Source::sqlite passes now. Client::Sync::sqlite still had some issues,
partly network timeouts due to load on the server, partly problems in the client (?).


git-svn-id: https://zeitsenke.de/svn/SyncEvolution/trunk@496 15ad00c4-1369-45f4-8270-35d70d36bdcd

src/Makefile.am

@@ -182,10 +182,12 @@ $(filter $(LOCAL_TEST_FILES), testcases) : % : $(srcdir)/../test/%
 	rm -rf $@
 	cp -r $< $@
 
-# verbatim files from client library
+# verbatim files from client library:
+# - SQLiteContactSource does not support all fields: filter those out
 $(filter-out $(LOCAL_TEST_FILES), testcases) : % : $(SYNC4J_SUBDIR)/test/test/%
 	rm -rf $@
 	cp -r $< $@
+	perl -e '$$_ = join("", <>); s/^(ADR|TEL|EMAIL|PHOTO).*?(?=^\S)//msg; print;' testcases/vcard21.vcf >testcases/vcard21_sqlite.vcf
 
 test : client-test testcases synccompare
 

src/SQLiteContactSource.cpp

@@ -42,10 +42,30 @@ enum {
     PERSON_FIRSTSORT,
     PERSON_ORGANIZATION,
     PERSON_DEPARTMENT,
+    PERSON_UNIT,
     PERSON_NOTE,
     PERSON_BIRTHDAY,
     PERSON_JOBTITLE,
+    PERSON_TITLE,
     PERSON_NICKNAME,
+    PERSON_FULLNAME,
+    PERSON_CATEGORIES,
+
+    PERSON_AIM,
+    PERSON_GROUPWISE,
+    PERSON_ICQ,
+    PERSON_YAHOO,
+
+    PERSON_FILEAS,
+    PERSON_ANNIVERSARY,
+
+    PERSON_ASSISTANT,
+    PERSON_MANAGER,
+    PERSON_SPOUSE,
+
+    PERSON_URL,
+    PERSON_BLOG_URL,
+    PERSON_VIDEO_URL,
 
     LAST_COL
 };
@@ -62,44 +82,78 @@ void SQLiteContactSource::open()
         { "LastSort", "ABPerson" },
         { "Organization", "ABPerson" },
         { "Department", "ABPerson" },
+        { "Unit", "ABPerson" },
         { "Note", "ABPerson", "NOTE" },
-        { "Birthday", "ABPerson", "BIRTHDAY" },
+        { "Birthday", "ABPerson", "BDAY" },
         { "JobTitle", "ABPerson", "ROLE" },
+        { "Title", "ABPerson", "TITLE" },
         { "Nickname", "ABPerson", "NICKNAME" },
+        { "CompositeNameFallback", "ABPerson", "FN" },
+        { "Categories", "ABPerson", "CATEGORIES" },
+
+        { "AIM", "ABPerson", "X-AIM" },
+        { "Groupwise", "ABPerson", "X-GROUPWISE" },
+        { "ICQ", "ABPerson", "X-ICQ" },
+        { "Yahoo", "ABPerson", "X-YAHOO" },
+
+        { "FileAs", "ABPerson", "X-EVOLUTION-FILE-AS" },
+        { "Anniversary", "ABPerson", "X-EVOLUTION-ANNIVERSARY" },
+
+        { "Assistant", "ABPerson", "X-EVOLUTION-ASSISTANT" },
+        { "Manager", "ABPerson", "X-EVOLUTION-MANAGER" },
+        { "Spouse", "ABPerson", "X-EVOLUTION-SPOUSE" },
+
+    
+        { "URL", "ABPerson", "URL" },
+        { "BlogURL", "ABPerson", "X-EVOLUTION-BLOG-URL" },
+        { "VideoURL", "ABPerson", "X-EVOLUTION-VIDEO-URL" },
+
         { NULL }
     };
     static const char *schema = 
         "BEGIN TRANSACTION;"
-        "CREATE TABLE ABMultiValue (UID INTEGER PRIMARY KEY, record_id INTEGER, property INTEGER, identifier INTEGER, label INTEGER, value TEXT);"
+        "CREATE TABLE ABPerson (ROWID INTEGER PRIMARY KEY AUTOINCREMENT, "
-        "CREATE TABLE ABMultiValueEntry (parent_id INTEGER, key INTEGER, value TEXT, UNIQUE(parent_id, key));"
+        "First TEXT, "
-        "CREATE TABLE ABMultiValueEntryKey (value TEXT, UNIQUE(value));"
+        "Last TEXT, "
-        "CREATE TABLE ABMultiValueLabel (value TEXT, UNIQUE(value));"
+        "Middle TEXT, "
-        "CREATE TABLE ABPerson (ROWID INTEGER PRIMARY KEY AUTOINCREMENT, First TEXT, Last TEXT, Middle TEXT, FirstPhonetic TEXT, MiddlePhonetic TEXT, LastPhonetic TEXT, Organization TEXT, Department TEXT, Note TEXT, Kind INTEGER, Birthday TEXT, JobTitle TEXT, Nickname TEXT, Prefix TEXT, Suffix TEXT, FirstSort TEXT, LastSort TEXT, CreationDate INTEGER, ModificationDate INTEGER, CompositeNameFallback TEXT);"
+        "FirstPhonetic TEXT, "
-        "INSERT INTO ABMultiValueLabel VALUES('_$!<Mobile>!$_');"
+        "MiddlePhonetic TEXT, "
-        "INSERT INTO ABMultiValueLabel VALUES('_$!<Home>!$_');"
+        "LastPhonetic TEXT, "
-        "INSERT INTO ABMultiValueLabel VALUES('_$!<Work>!$_');"
+        "Organization TEXT, "
-        "INSERT INTO ABMultiValueEntryKey VALUES('CountryCode');"
+        "Department TEXT, "
-        "INSERT INTO ABMultiValueEntryKey VALUES('City');"
+        "Unit TEXT, "
-        "INSERT INTO ABMultiValueEntryKey VALUES('Street');"
+        "Note TEXT, "
-        "INSERT INTO ABMultiValueEntryKey VALUES('State');"
+        "Kind INTEGER, "
-        "INSERT INTO ABMultiValueEntryKey VALUES('ZIP');"
+        "Birthday TEXT, "
-        "INSERT INTO ABMultiValueEntryKey VALUES('Country');"
+        "JobTitle TEXT, "
+        "Title TEXT, "
+        "Nickname TEXT, "
+        "Prefix TEXT, "
+        "Suffix TEXT, "
+        "FirstSort TEXT, "
+        "LastSort TEXT, "
+        "CreationDate INTEGER, "
+        "ModificationDate INTEGER, "
+        "CompositeNameFallback TEXT, "
+        "Categories TEXT, "
+        "AIM TEXT, "
+        "Groupwise TEXT, "
+        "ICQ Text, "
+        "Yahoo TEXT, "
+        "Anniversary TEXT, "
+        "Assistant TEXT, "
+        "Manager TEXT, "
+        "Spouse TEXT, "
+        "URL TEXT, "
+        "BlogURL TEXT, "
+        "VideoURL TEXT, "
+        "FileAs TEXT);"
         "COMMIT;";
 
     m_sqlite.open(getName(),
                   m_id,
                   mapping,
                   schema);
-
-    // query database for certain constant indices
-    m_addrCountryCode = m_sqlite.findKey("ABMultiValueEntryKey", "value", "CountryCode");
-    m_addrCity = m_sqlite.findKey("ABMultiValueEntryKey", "value", "City");
-    m_addrStreet = m_sqlite.findKey("ABMultiValueEntryKey", "value", "Street");
-    m_addrState = m_sqlite.findKey("ABMultiValueEntryKey", "value", "State");
-    m_addrZIP = m_sqlite.findKey("ABMultiValueEntryKey", "value", "ZIP");
-    m_typeMobile = m_sqlite.findKey("ABMultiValueLabel", "value", "_$!<Mobile>!$_");
-    m_typeHome = m_sqlite.findKey("ABMultiValueLabel", "value", "_$!<Home>!$_");
-    m_typeWork = m_sqlite.findKey("ABMultiValueLabel", "value", "_$!<Work>!$_");
 }
 
 void SQLiteContactSource::close()
@@ -139,14 +193,20 @@ SyncItem *SQLiteContactSource::createItem(const string &uid)
     tmp += m_sqlite.getTextColumn(contact, m_sqlite.getMapping(PERSON_MIDDLE).colindex);
     tmp += VObject::SEMICOLON_REPLACEMENT;
     tmp += m_sqlite.getTextColumn(contact, m_sqlite.getMapping(PERSON_FIRST).colindex);
-    if (tmp.size() > 2) {
+    tmp += VObject::SEMICOLON_REPLACEMENT;
+    tmp += m_sqlite.getTextColumn(contact, m_sqlite.getMapping(PERSON_PREFIX).colindex);
+    tmp += VObject::SEMICOLON_REPLACEMENT;
+    tmp += m_sqlite.getTextColumn(contact, m_sqlite.getMapping(PERSON_SUFFIX).colindex);
+    if (tmp.size() > 4) {
         vobj.addProperty("N", tmp.c_str());
     }
 
     tmp = m_sqlite.getTextColumn(contact, m_sqlite.getMapping(PERSON_ORGANIZATION).colindex);
     tmp += VObject::SEMICOLON_REPLACEMENT;
     tmp += m_sqlite.getTextColumn(contact, m_sqlite.getMapping(PERSON_DEPARTMENT).colindex);
-    if (tmp.size() > 1) {
+    tmp += VObject::SEMICOLON_REPLACEMENT;
+    tmp += m_sqlite.getTextColumn(contact, m_sqlite.getMapping(PERSON_UNIT).colindex);
+    if (tmp.size() > 2) {
         vobj.addProperty("ORG", tmp.c_str());
     }
  
@@ -174,11 +234,34 @@ string SQLiteContactSource::insertItem(string &uid, const SyncItem &item)
     }
     vobj->toNativeEncoding();
 
+    int numparams = 0;
     stringstream cols;
     stringstream values;
     VProperty *prop;
 
-    // always set the name, even if not in the vcard
+    // parse up to three fields of ORG
+    prop = vobj->getProperty("ORG");
+    string organization, department, unit;
+    if (prop && prop->getValue()) {
+        string fn = prop->getValue();
+        size_t sep1 = fn.find(VObject::SEMICOLON_REPLACEMENT);
+        size_t sep2 = sep1 == fn.npos ? fn.npos : fn.find(VObject::SEMICOLON_REPLACEMENT, sep1 + 1);
+
+        organization = fn.substr(0, sep1);
+        if (sep1 != fn.npos) {
+            department = fn.substr(sep1 + 1, (sep2 == fn.npos) ? fn.npos : sep2 - sep1 - 1);
+        }
+        if (sep2 != fn.npos) {
+            unit = fn.substr(sep2 + 1);
+        }
+    }
+    cols << m_sqlite.getMapping(PERSON_ORGANIZATION).colname << ", " <<
+        m_sqlite.getMapping(PERSON_DEPARTMENT).colname << ", " <<
+        m_sqlite.getMapping(PERSON_UNIT).colname << ", ";
+    values << "?, ?, ?, ";
+    numparams += 3;
+
+    // parse the name, insert empty fields if not found
     prop = vobj->getProperty("N");
     string first, middle, last, prefix, suffix, firstsort, lastsort;
     if (prop && prop->getValue()) {
@@ -205,9 +288,13 @@ string SQLiteContactSource::insertItem(string &uid, const SyncItem &item)
     cols << m_sqlite.getMapping(PERSON_FIRST).colname << ", " <<
         m_sqlite.getMapping(PERSON_MIDDLE).colname << ", " <<
         m_sqlite.getMapping(PERSON_LAST).colname << ", " <<
+        m_sqlite.getMapping(PERSON_PREFIX).colname << ", " <<
+        m_sqlite.getMapping(PERSON_SUFFIX).colname << ", " <<
         m_sqlite.getMapping(PERSON_LASTSORT).colname << ", " <<
         m_sqlite.getMapping(PERSON_FIRSTSORT).colname;
-    values << "?, ?, ?, ?, ?";
+    values << "?, ?, ?, ?, ?, ?, ?";
+    numparams += 7;
+
 
     // synthesize sort keys: upper case with specific order of first/last name
     firstsort = first + " " + last;
@@ -220,9 +307,11 @@ string SQLiteContactSource::insertItem(string &uid, const SyncItem &item)
         creationTime = m_sqlite.findColumn("ABPerson", "ROWID", uid.c_str(), "CreationDate", "");
         cols << ", ROWID";
         values << ", ?";
+        numparams++;
     }
     cols << ", CreationDate, ModificationDate";
     values << ", ?, ?";
+    numparams += 2;
 
     // delete complete row so that we can recreate it
     if (uid.size()) {
@@ -233,16 +322,22 @@ string SQLiteContactSource::insertItem(string &uid, const SyncItem &item)
 
     string cols_str = cols.str();
     string values_str = values.str();
-    eptr<sqlite3_stmt> insert(m_sqlite.prepareSQL("INSERT INTO ABPerson( %s ) "
+    eptr<sqlite3_stmt> insert(m_sqlite.vObjectToRow(*vobj,
-                                                  "VALUES( %s );",
+                                                    "ABPerson",
-                                                  cols_str.c_str(),
+                                                    numparams,
-                                                  values_str.c_str()));
+                                                    cols.str(),
+                                                    values.str()));
 
     // now bind parameter values in the same order as the columns specification above
     int param = 1;
+    m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, organization.c_str(), -1, SQLITE_TRANSIENT));
+    m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, department.c_str(), -1, SQLITE_TRANSIENT));
+    m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, unit.c_str(), -1, SQLITE_TRANSIENT));
     m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, first.c_str(), -1, SQLITE_TRANSIENT));
     m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, middle.c_str(), -1, SQLITE_TRANSIENT));
     m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, last.c_str(), -1, SQLITE_TRANSIENT));
+    m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, prefix.c_str(), -1, SQLITE_TRANSIENT));
+    m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, suffix.c_str(), -1, SQLITE_TRANSIENT));
     m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, lastsort.c_str(), -1, SQLITE_TRANSIENT));
     m_sqlite.checkSQL(sqlite3_bind_text(insert, param++, firstsort.c_str(), -1, SQLITE_TRANSIENT));
     if (uid.size()) {
@@ -266,21 +361,8 @@ string SQLiteContactSource::insertItem(string &uid, const SyncItem &item)
 
 void SQLiteContactSource::deleteItem(const string &uid)
 {
-    // delete address field members of contact
+    eptr<sqlite3_stmt> del;
-    eptr<sqlite3_stmt> del(m_sqlite.prepareSQL("DELETE FROM ABMultiValueEntry "
-                                               "WHERE ABMultiValueEntry.parent_id IN "
-                                               "(SELECT ABMultiValue.uid FROM ABMultiValue WHERE "
-                                               " ABMultiValue.record_id = ?);"));
-    m_sqlite.checkSQL(sqlite3_bind_text(del, 1, uid.c_str(), -1, SQLITE_TRANSIENT));
-    m_sqlite.checkSQL(sqlite3_step(del));
 
-    // delete addresses and emails of contact
-    del.set(m_sqlite.prepareSQL("DELETE FROM ABMultiValue WHERE "
-                                "ABMultiValue.record_id = ?;"));
-    m_sqlite.checkSQL(sqlite3_bind_text(del, 1, uid.c_str(), -1, SQLITE_TRANSIENT));
-    m_sqlite.checkSQL(sqlite3_step(del));
-
-    // now delete the contact itself
     del.set(m_sqlite.prepareSQL("DELETE FROM ABPerson WHERE "
                                 "ABPerson.ROWID = ?;"));
     m_sqlite.checkSQL(sqlite3_bind_text(del, 1, uid.c_str(), -1, SQLITE_TRANSIENT));

src/SQLiteContactSource.h

@@ -26,11 +26,25 @@
 #ifdef ENABLE_SQLITE
 
 /**
- * Uses SQLiteUtil for contacts
+ * Uses SQLiteUtil for contacts with a schema inspired by the one used
- * with a schema as used by Mac OS X.
+ * by Mac OS X.  That schema has hierarchical tables which is not
- * The schema has hierarchical tables, which is not
+ * supported by SQLiteUtil, therefore SQLiteContactSource uses a
- * supported by SQLiteUtil, so only the properties which
+ * simplified schema where each contact consists of one row in the
- * have a 1:1 mapping are currently stored.
+ * database table.
+ *
+ * The handling of the "N" and "ORG" property shows how mapping
+ * between one property and multiple different columns works.
+ *
+ * Properties which can occur more than once per contact like address,
+ * email and phone numbers are not supported. They would have to be
+ * stored in additional tables.
+ *
+ * Change tracking is done by implementing a modification date as part
+ * of each contact and using that as the revision string required by
+ * TrackingSyncSource, which then takes care of change tracking.
+ *
+ * The database file is created automatically if the database ID is
+ * file:///<path>.
  */
 class SQLiteContactSource : public TrackingSyncSource
 {
@@ -61,17 +75,6 @@ class SQLiteContactSource : public TrackingSyncSource
  private:
     /** encapsulates access to database */
     SQLiteUtil m_sqlite;
-
-    /** constant key values defined by tables in the database, queried during open() */
-    key_t m_addrCountryCode,
-        m_addrCity,
-        m_addrStreet,
-        m_addrState,
-        m_addrZIP,
-        m_addrCountry,
-        m_typeMobile,
-        m_typeHome,
-        m_typeWork;
 };
 
 #endif // ENABLE_SQLITE

src/SQLiteUtil.cpp

@@ -26,6 +26,7 @@
 #include "vocl/VConverter.h"
 
 #include <stdarg.h>
+#include <sstream>
 
 void SQLiteUtil::throwError(const string &operation)
 {
@@ -128,6 +129,62 @@ void SQLiteUtil::rowToVObject(sqlite3_stmt *stmt, vocl::VObject &vobj)
     }
 }
 
+sqlite3_stmt *SQLiteUtil::vObjectToRow(vocl::VObject &vobj,
+                                       const string &tablename,
+                                       int numparams,
+                                       const string &cols,
+                                       const string &values)
+{
+    stringstream cols_stream;
+    stringstream values_stream;
+    int i;
+
+    cols_stream << cols;
+    values_stream << values;
+
+    // figure out which columns we will fill
+    for (i = 0; m_mapping[i].colname; i++) {
+        if (m_mapping[i].colindex < 0 ||
+            !m_mapping[i].propname ||
+            tablename != m_mapping[i].tablename) {
+            continue;
+        }
+        
+        vocl::VProperty *vprop = vobj.getProperty(m_mapping[i].propname);
+        if (vprop) {
+            if (cols.size()) {
+                cols_stream << ", ";
+                values_stream << ", ";
+            }
+            cols_stream << m_mapping[i].colname;
+            values_stream << "?";
+        }
+    }
+
+    // create statement
+    eptr<sqlite3_stmt> insert(prepareSQL("INSERT INTO ABPerson( %s ) "
+                                         "VALUES( %s );",
+                                         cols_stream.str().c_str(),
+                                         values_stream.str().c_str()));
+    // now bind our parameters
+    int param = numparams + 1;
+    for (i = 0; m_mapping[i].colname; i++) {
+        if (m_mapping[i].colindex < 0 ||
+            !m_mapping[i].propname ||
+            tablename != m_mapping[i].tablename) {
+            continue;
+        }
+        
+        vocl::VProperty *vprop = vobj.getProperty(m_mapping[i].propname);
+        if (vprop) {
+            const char *text = vprop->getValue();
+            checkSQL(sqlite3_bind_text(insert, param++, text ? text : "", -1, SQLITE_TRANSIENT));
+        }
+    }
+
+    return insert.release();
+}
+
 void SQLiteUtil::open(const string &name,
                       const string &fileid,
                       const SQLiteUtil::Mapping *mapping,

src/SQLiteUtil.h

@@ -124,6 +124,19 @@ class SQLiteUtil
     /** copies all columns which directly map to a property into the vobj */
     void rowToVObject(sqlite3_stmt *stmt, vocl::VObject &vobj);
 
+    /**
+     * Creates a SQL INSERT INTO <tablename> ( <cols> ) VALUES ( <values> )
+     * statement and binds all rows/values that map directly from the vobj.
+     *
+     * @param numparams      number of ? placeholders in values; the caller has
+     *                       to bind those before executing the statement
+     */
+    sqlite3_stmt *vObjectToRow(vocl::VObject &vobj,
+                               const string &tablename,
+                               int numparams,
+                               const string &cols,
+                               const string &values);
+
  private:
     /* copy of open() parameters */
     arrayptr<Mapping> m_mapping;

src/client-test-app.cpp

@@ -400,6 +400,7 @@ public:
             getTestData("vcard21", config);
             config.sourceName = "sqlite";
             config.type = "sqlite";
+            config.testcases = "testcases/vcard21_sqlite.vcf";
             break;
          case TEST_ADDRESS_BOOK_SOURCE:
             getTestData("vcard30", config);