added --restore and fixed --status

--status was broken by the last commit: XDG_DATA_DIR was not
expanded. Fixed it as part of the improvements for printing
changes in --restore. Also made some other minor changes
to utility classes as part of these improvements.

The new --restore option can restore data from a data dump,
identified by the directory and a before/after flag. The
corresponding API is EvolutionSyncClient::restore().
This commit is contained in:
Patrick Ohly 2009-04-23 16:47:07 +02:00
parent e47a9225bc
commit ad774d0b29
10 changed files with 307 additions and 40 deletions

View file

@ -84,7 +84,7 @@ public:
virtual SyncItem *createItem(const string &uid) { return m_source->createItem(uid); }
virtual void close() { m_source->close(); }
virtual void backupData(const string &dir, ConfigNode &node, BackupReport &report) { m_source->backupData(dir, node, report); }
virtual void restoreData(const string &dir, const ConfigNode &node) { m_source->restoreData(dir, node); }
virtual void restoreData(const string &dir, const ConfigNode &node, bool dryrun, SyncSourceReport &report) { m_source->restoreData(dir, node, dryrun, report); }
virtual const char *getMimeType() const { return m_source->getMimeType(); }
virtual const char *getMimeVersion() const { return m_source->getMimeVersion(); }
virtual const char* getSupportedTypes() const { return m_source->getSupportedTypes(); }

View file

@ -50,6 +50,7 @@ EvolutionSyncClient::EvolutionSyncClient(const string &server,
m_sources(sources),
m_doLogging(doLogging),
m_quiet(false),
m_dryrun(false),
m_engine(new sysync::TEngineModuleBridge())
{
// Use libsynthesis that we were linked against. The name of a
@ -294,7 +295,8 @@ public:
// @param logLevel 0 = default, 1 = ERROR, 2 = INFO, 3 = DEBUG
// @param usePath write directly into path, don't create and manage subdirectories
// @param report record information about session here (may be NULL)
void startSession(const char *path, int maxlogdirs, int logLevel, bool usePath, SyncReport *report) {
// @param logname the basename to be used for logs, traditionally "client" for syncs
void startSession(const char *path, int maxlogdirs, int logLevel, bool usePath, SyncReport *report, const string &logname) {
m_maxlogdirs = maxlogdirs;
m_report = report;
if (path && !strcasecmp(path, "none")) {
@ -335,14 +337,14 @@ public:
}
}
} else {
m_path = path;
m_path = m_logdir;
if (mkdir(m_path.c_str(), S_IRWXU) &&
errno != EEXIST) {
SE_LOG_DEBUG(NULL, NULL, "%s: %s", m_path.c_str(), strerror(errno));
EvolutionSyncClient::throwError(m_path, errno);
}
}
m_logfile = m_path + "/client.log";
m_logfile = m_path + "/" + logname + ".log";
}
if (m_logfile.size()) {
@ -550,18 +552,11 @@ public:
*/
void dumpDatabases(const string &suffix,
BackupReport SyncSourceReport::*report) {
ofstream out;
#ifndef IPHONE
// output stream on iPhone raises exception even though it is in a good state;
// perhaps the missing C++ exception support is the reason:
// http://code.google.com/p/iphone-dev/issues/detail?id=48
out.exceptions(ios_base::badbit|ios_base::failbit|ios_base::eofbit);
#endif
BOOST_FOREACH(EvolutionSyncSource *source, *this) {
string dir = databaseName(*source, suffix);
boost::shared_ptr<ConfigNode> node = ConfigNode::createFileNode(dir + ".ini");
SE_LOG_DEBUG(NULL, NULL, "creating %s", dir.c_str());
rm_r(dir);
mkdir_p(dir);
BackupReport dummy;
source->backupData(dir, *node,
@ -570,6 +565,16 @@ public:
}
}
void restoreDatabase(EvolutionSyncSource &source, const string &suffix, bool dryrun, SyncSourceReport &report)
{
string dir = databaseName(source, suffix);
boost::shared_ptr<ConfigNode> node = ConfigNode::createFileNode(dir + ".ini");
if (!node->exists()) {
EvolutionSyncClient::throwError(dir + ": no such database backup found");
}
source.restoreData(dir, *node, dryrun, report);
}
SourceList(const string &server, bool doLogging) :
m_logdir(server),
m_prepared(false),
@ -580,16 +585,17 @@ public:
}
// call as soon as logdir settings are known
void startSession(const char *logDirPath, int maxlogdirs, int logLevel, SyncReport *report) {
void startSession(const char *logDirPath, int maxlogdirs, int logLevel, SyncReport *report,
const string &logname) {
m_previousLogdir = m_logdir.previousLogdir(logDirPath);
if (m_doLogging) {
m_logdir.startSession(logDirPath, maxlogdirs, logLevel, false, report);
m_logdir.startSession(logDirPath, maxlogdirs, logLevel, false, report, logname);
} else {
// Run debug session without paying attention to
// the normal logdir handling. The log level here
// refers to stdout. The log file will be as complete
// as possible.
m_logdir.startSession(logDirPath, 0, 1, true, report);
m_logdir.startSession(logDirPath, 0, 1, true, report, logname);
}
}
@ -605,24 +611,27 @@ public:
void setPath(const string &path) { m_logdir.setPath(path); }
/**
* If possible (m_previousLogdir found) and enabled,
* If possible (directory to compare against available) and enabled,
* then dump changes applied locally.
*
* @param oldSuffix suffix of old database dump: usually "after"
* @param currentSuffix the current database dump suffix: "current"
* when not doing a sync, otherwise "before"
*/
bool dumpLocalChanges(const string &oldSuffix, const string &newSuffix) {
if (m_logLevel <= LOGGING_SUMMARY || !m_previousLogdir.size()) {
bool dumpLocalChanges(const string &oldDir,
const string &oldSuffix, const string &newSuffix,
const string &intro = "Local changes to be applied to server during synchronization:\n",
const string &config = "CLIENT_TEST_LEFT_NAME='after last sync' CLIENT_TEST_RIGHT_NAME='current data' CLIENT_TEST_REMOVED='removed since last sync' CLIENT_TEST_ADDED='added since last sync'") {
if (m_logLevel <= LOGGING_SUMMARY || oldDir.empty()) {
return false;
}
cout << "Local changes to be applied to server during synchronization:\n";
cout << intro;
BOOST_FOREACH(EvolutionSyncSource *source, *this) {
string oldFile = databaseName(*source, oldSuffix, m_previousLogdir);
string oldFile = databaseName(*source, oldSuffix, oldDir);
string newFile = databaseName(*source, newSuffix);
cout << "*** " << source->getName() << " ***\n" << flush;
string cmd = string("env CLIENT_TEST_COMPARISON_FAILED=10 CLIENT_TEST_LEFT_NAME='after last sync' CLIENT_TEST_RIGHT_NAME='current data' CLIENT_TEST_REMOVED='removed since last sync' CLIENT_TEST_ADDED='added since last sync' synccompare 2>/dev/null '" ) +
string cmd = string("env CLIENT_TEST_COMPARISON_FAILED=10 " + config + " synccompare 2>/dev/null '" ) +
oldFile + "' '" + newFile + "'";
int ret = system(cmd.c_str());
switch (ret == -1 ? ret : WEXITSTATUS(ret)) {
@ -648,7 +657,7 @@ public:
// dump initial databases
dumpDatabases("before", &SyncSourceReport::m_backupBefore);
// compare against the old "after" database dump
dumpLocalChanges("after", "before");
dumpLocalChanges(getPrevLogdir(), "after", "before");
m_prepared = true;
}
@ -1370,7 +1379,8 @@ SyncMLStatus EvolutionSyncClient::sync(SyncReport *report)
sourceList.startSession(getLogDir(),
getMaxLogDirs(),
getLogLevel(),
report);
report,
"client");
// dump some summary information at the beginning of the log
SE_LOG_DEV(NULL, NULL, "SyncML server account: %s", getUsername());
@ -1777,7 +1787,7 @@ void EvolutionSyncClient::status()
source->open();
}
sourceList.startSession(getLogDir(), 0, 0, NULL);
sourceList.startSession(getLogDir(), 0, 0, NULL, "status");
LoggerBase::instance().setLevel(Logger::INFO);
string prevLogdir = sourceList.getPrevLogdir();
bool found = access(prevLogdir.c_str(), R_OK|X_OK) == 0;
@ -1786,7 +1796,7 @@ void EvolutionSyncClient::status()
try {
sourceList.setPath(prevLogdir);
sourceList.dumpDatabases("current", NULL);
sourceList.dumpLocalChanges("after", "current");
sourceList.dumpLocalChanges(sourceList.getPrevLogdir(), "after", "current");
} catch(...) {
SyncEvolutionException::handle();
}
@ -1798,6 +1808,75 @@ void EvolutionSyncClient::status()
}
}
static void logRestoreReport(const SyncReport &report, bool dryrun)
{
if (!report.empty()) {
stringstream out;
out << report;
SE_LOG_INFO(NULL, NULL, "Item changes %s applied to client during restore:\n%s",
dryrun ? "to be" : "that were",
out.str().c_str());
SE_LOG_INFO(NULL, NULL, "The same incremental changes will be applied to the server during the next sync.");
SE_LOG_INFO(NULL, NULL, "Use -sync refresh-from-client to replace the complete data on the server.");
}
}
void EvolutionSyncClient::restore(const string &dirname, RestoreDatabase database)
{
if (!exists()) {
SE_LOG_ERROR(NULL, NULL, "No configuration for server \"%s\" found.", m_server.c_str());
throwError("cannot proceed without configuration");
}
SourceList sourceList(m_server, false);
sourceList.startSession(dirname.c_str(), 0, 0, NULL, "restore");
LoggerBase::instance().setLevel(Logger::INFO);
initSources(sourceList);
BOOST_FOREACH(EvolutionSyncSource *source, sourceList) {
source->checkPassword(*this);
}
string datadump = database == DATABASE_BEFORE_SYNC ? "before" : "after";
BOOST_FOREACH(EvolutionSyncSource *source, sourceList) {
source->open();
}
if (!m_quiet) {
sourceList.dumpDatabases("current", NULL);
sourceList.dumpLocalChanges(dirname, "current", datadump,
"Data changes to be applied to local data during restore:\n",
"CLIENT_TEST_LEFT_NAME='current data' "
"CLIENT_TEST_REMOVED='after restore' "
"CLIENT_TEST_REMOVED='to be removed' "
"CLIENT_TEST_ADDED='to be added'");
}
SyncReport report;
try {
BOOST_FOREACH(EvolutionSyncSource *source, sourceList) {
SyncSourceReport sourcereport;
try {
SE_LOG_DEBUG(NULL, NULL, "Restoring %s...", source->getName());
sourceList.restoreDatabase(*source,
datadump,
m_dryrun,
sourcereport);
SE_LOG_DEBUG(NULL, NULL, "... %s restored.", source->getName());
report.addSyncSourceReport(source->getName(), sourcereport);
} catch (...) {
sourcereport.recordStatus(STATUS_FATAL);
report.addSyncSourceReport(source->getName(), sourcereport);
throw;
}
}
} catch (...) {
logRestoreReport(report, m_dryrun);
throw;
}
logRestoreReport(report, m_dryrun);
}
void EvolutionSyncClient::getSessions(vector<string> &dirs)
{
LogDir logging(m_server);

View file

@ -43,6 +43,7 @@ class EvolutionSyncClient : public EvolutionSyncConfig, public ConfigUserInterfa
const set<string> m_sources;
const bool m_doLogging;
bool m_quiet;
bool m_dryrun;
/**
* a pointer to the active SourceList instance if one exists;
@ -91,6 +92,9 @@ class EvolutionSyncClient : public EvolutionSyncConfig, public ConfigUserInterfa
bool getQuiet() { return m_quiet; }
void setQuiet(bool quiet) { m_quiet = quiet; }
bool getDryRun() { return m_dryrun; }
void setDryRun(bool dryrun) { m_dryrun = dryrun; }
/**
* Executes the sync, throws an exception in case of failure.
* Handles automatic backups and report generation.
@ -106,6 +110,17 @@ class EvolutionSyncClient : public EvolutionSyncConfig, public ConfigUserInterfa
*/
void status();
enum RestoreDatabase {
DATABASE_BEFORE_SYNC,
DATABASE_AFTER_SYNC
};
/**
* Restore data of selected sources from before or after the given
* sync session, identified by absolute path to the log dir.
*/
void restore(const string &dirname, RestoreDatabase database);
/**
* fills vector with absolute path to information about previous
* sync sessions, oldest one first

View file

@ -369,9 +369,10 @@ class EvolutionSyncSource : public EvolutionSyncSourceConfig, public LoggerBase,
virtual void backupData(const string &dirname, ConfigNode &node, BackupReport &report) = 0;
/**
* Restore database from data stored in backupData().
* Restore database from data stored in backupData(). Will be
* called inside open()/close() pair. beginSync() is *not* called.
*/
virtual void restoreData(const string &dirname, const ConfigNode &node) = 0;
virtual void restoreData(const string &dirname, const ConfigNode &node, bool dryrun, SyncSourceReport &report) = 0;
/**
* Returns the preferred mime type of the items handled by the sync source.

View file

@ -98,6 +98,27 @@ bool SyncEvolutionCmdline::parse()
} else if(boost::iequals(m_argv[opt], "--run") ||
boost::iequals(m_argv[opt], "-r")) {
m_run = true;
} else if(boost::iequals(m_argv[opt], "--restore")) {
opt++;
if (opt >= m_argc) {
usage(true, string("missing parameter for ") + cmdOpt(m_argv[opt - 1]));
return false;
}
m_restore = m_argv[opt];
if (m_restore.empty()) {
usage(true, string("missing parameter for ") + cmdOpt(m_argv[opt - 1]));
return false;
}
if (!isDir(m_restore)) {
usage(true, string("parameter '") + m_restore + "' for " + cmdOpt(m_argv[opt - 1]) + " must be log directory");
return false;
}
} else if(boost::iequals(m_argv[opt], "--before")) {
m_before = true;
} else if(boost::iequals(m_argv[opt], "--after")) {
m_after = true;
} else if(boost::iequals(m_argv[opt], "--dry-run")) {
m_dryrun = true;
} else if(boost::iequals(m_argv[opt], "--migrate")) {
m_migrate = true;
} else if(boost::iequals(m_argv[opt], "--status") ||
@ -129,6 +150,11 @@ bool SyncEvolutionCmdline::parse()
}
bool SyncEvolutionCmdline::run() {
// --dry-run is only supported by some operations.
// Be very strict about it and make sure it is off in all
// potentially harmful operations, otherwise users might
// expect it to have an effect when it doesn't.
if (m_usage) {
usage(true);
} else if (m_version) {
@ -140,6 +166,7 @@ bool SyncEvolutionCmdline::run() {
} else if (m_dontrun) {
// user asked for information
} else if (m_argc == 1) {
// no parameters: list databases and short usage
const SourceRegistry &registry(EvolutionSyncSource::getSourceRegistry());
boost::shared_ptr<FilterConfigNode> configNode(new VolatileConfigNode());
boost::shared_ptr<FilterConfigNode> hiddenNode(new VolatileConfigNode());
@ -206,6 +233,10 @@ bool SyncEvolutionCmdline::run() {
usage(true, "server name missing");
return false;
} else if (m_configure || m_migrate) {
if (m_dryrun) {
EvolutionSyncClient::throwError("--dry-run not supported for configuration changes");
}
bool fromScratch = false;
// Both config changes and migration are implemented as copying from
@ -330,6 +361,10 @@ bool SyncEvolutionCmdline::run() {
// done, now write it
to->flush();
} else if (m_remove) {
if (m_dryrun) {
EvolutionSyncClient::throwError("--dry-run not supported for removing configurations");
}
// extra sanity check
if (!m_sources.empty() ||
!m_syncProps.empty() ||
@ -345,6 +380,7 @@ bool SyncEvolutionCmdline::run() {
} else {
EvolutionSyncClient client(m_server, true, m_sources);
client.setQuiet(m_quiet);
client.setDryRun(m_dryrun);
client.setConfigFilter(true, m_syncProps);
client.setConfigFilter(false, m_sourceProps);
if (m_status) {
@ -366,7 +402,26 @@ bool SyncEvolutionCmdline::run() {
cout << report;
}
}
} else if (!m_restore.empty()) {
// sanity checks: either --after or --before must be given, sources must be selected
if ((!m_after && !m_before) ||
(m_after && m_before)) {
usage(false, "--restore <log dir> must be used with either --after (restore database as it was after that sync) or --before (restore data from before sync)");
return false;
}
if (m_sources.empty()) {
usage(false, "Sources must be selected explicitly for --restore to prevent accidental restore.");
return false;
}
client.restore(m_restore,
m_after ?
EvolutionSyncClient::DATABASE_AFTER_SYNC :
EvolutionSyncClient::DATABASE_BEFORE_SYNC);
} else {
if (m_dryrun) {
EvolutionSyncClient::throwError("--dry-run not supported for running a synchronization");
}
// safety catch: if props are given, then --run
// is required
if (!m_run &&
@ -577,6 +632,8 @@ void SyncEvolutionCmdline::usage(bool full, const string &error, const string &p
out << "Run a synchronization:" << endl;
out << " " << m_argv[0] << " <server> [<source> ...]" << endl;
out << " " << m_argv[0] << " --run <options for run> <server> [<source> ...]" << endl;
out << "Restore data from the automatic backups:" << endl;
out << " " << m_argv[0] << " --restore <session directory> --before|--after [--dry-run] <server> <source> ..." << endl;
out << "Remove a configuration:" << endl;
out << " " << m_argv[0] << " --remove <server>" << endl;
out << "Modify configuration:" << endl;
@ -636,6 +693,18 @@ void SyncEvolutionCmdline::usage(bool full, const string &error, const string &p
" --migrate implies --configure and can be combined with modifying" << endl <<
" properties." << endl <<
"" << endl <<
"--restore" << endl <<
" Restores the data of the selected sources to the state from before or after the" << endl <<
" selected synchronization. The synchronization is selected via its log directory" << endl <<
" (see --print-sessions). Other directories can also be given as long as" << endl <<
" they contain database dumps in the format created by SyncEvolution." << endl <<
" The output includes information about the changes made during the" << endl <<
" restore, both in terms of item changes and content changes (which is" << endl <<
" not always the same, see manual for details). This output can be suppressed" << endl <<
" with --quiet." << endl <<
" In combination with --dry-run, the changes to local data are only simulated." << endl <<
" This can be used to check that --restore will not remove valuable information." << endl <<
"" << endl <<
"--remove" << endl <<
" This removes only the configuration files and related meta information." << endl <<
" If other files were added to the config directory of the server, then" << endl <<

View file

@ -48,6 +48,7 @@ private:
ostream &m_out, &m_err;
Bool m_quiet;
Bool m_dryrun;
Bool m_status;
Bool m_version;
Bool m_usage;
@ -63,6 +64,9 @@ private:
const ConfigPropertyRegistry &m_validSyncProps;
const ConfigPropertyRegistry &m_validSourceProps;
string m_restore;
Bool m_before, m_after;
string m_server;
string m_template;
set<string> m_sources;

View file

@ -180,17 +180,25 @@ std::ostream &operator << (std::ostream &out, const SyncReport &report)
std::stringstream line;
line <<
PrettyPrintSyncMode(source.getFinalSyncMode()) << ", " <<
if (source.getFinalSyncMode() != SYNC_NONE ||
source.getItemStat(SyncSourceReport::ITEM_LOCAL,
SyncSourceReport::ITEM_ANY,
SyncSourceReport::ITEM_SENT_BYTES) / 1024 <<
" KB sent by client, " <<
SyncSourceReport::ITEM_SENT_BYTES) ||
source.getItemStat(SyncSourceReport::ITEM_LOCAL,
SyncSourceReport::ITEM_ANY,
SyncSourceReport::ITEM_RECEIVED_BYTES) / 1024 <<
" KB received";
flushRight(out, line.str());
SyncSourceReport::ITEM_RECEIVED_BYTES)) {
line <<
PrettyPrintSyncMode(source.getFinalSyncMode()) << ", " <<
source.getItemStat(SyncSourceReport::ITEM_LOCAL,
SyncSourceReport::ITEM_ANY,
SyncSourceReport::ITEM_SENT_BYTES) / 1024 <<
" KB sent by client, " <<
source.getItemStat(SyncSourceReport::ITEM_LOCAL,
SyncSourceReport::ITEM_ANY,
SyncSourceReport::ITEM_RECEIVED_BYTES) / 1024 <<
" KB received";
flushRight(out, line.str());
}
if (total_conflicts > 0) {
for (SyncSourceReport::ItemResult result = SyncSourceReport::ITEM_CONFLICT_SERVER_WON;

View file

@ -61,6 +61,7 @@ class SyncItem {
const char *getData() const { return m_data.c_str(); }
size_t getDataSize() const { return m_data.size(); }
void setData(const char *data, size_t size) { m_data.assign(data, size); }
void setData(const std::string &data) { m_data = data; }
void setDataType(const std::string &datatype) { m_datatype = datatype; }
std::string getDataType() const { return m_datatype; }
@ -199,6 +200,11 @@ class SyncSourceReport {
int count) {
m_stat[location][state][success] = count;
}
void incrementItemStat(ItemLocation location,
ItemState state,
ItemResult success) {
m_stat[location][state][success]++;
}
void recordFinalSyncMode(SyncMode mode) { m_mode = mode; }
SyncMode getFinalSyncMode() const { return m_mode; }

View file

@ -157,14 +157,98 @@ void TrackingSyncSource::backupData(const string &dir, ConfigNode &node, BackupR
report.setNumItems(counter - 1);
}
void TrackingSyncSource::restoreData(const string &dir, const ConfigNode &node)
void TrackingSyncSource::restoreData(const string &dir, const ConfigNode &node, bool dryrun, SyncSourceReport &report)
{
RevisionMap_t revisions;
listAllItems(revisions);
// int counter = 1;
// BOOST_FOREACH(const StringPair &mapping, revisions) {
// }
long numitems;
string strval;
strval = node.readProperty("numitems");
stringstream stream(strval);
stream >> numitems;
for (long counter = 1; counter <= numitems; counter++) {
stringstream key;
key << counter << "-uid";
string uid = node.readProperty(key.str());
key.clear();
key << counter << "-rev";
string rev = node.readProperty(key.str());
RevisionMap_t::iterator it = revisions.find(uid);
if (it != revisions.end() &&
it->second == rev) {
// item exists in backup and database with same revision:
// nothing to do
} else {
// add or update, so need item
SyncItem item;
stringstream filename;
filename << dir << "/" << counter;
string data;
if (!ReadFile(filename.str(), data)) {
throwError(StringPrintf("restoring %s from %s failed: could not read file",
uid.c_str(),
filename.str().c_str()));
}
item.setData(data);
item.setDataType("raw");
// TODO: it would be nicer to recreate the item
// with the original revision. If multiple peers
// synchronize against us, then some of them
// might still be in sync with that revision. By
// updating the revision here we force them to
// needlessly receive an update.
//
// For the current peer for which we restore this is
// avoided by the revision check above: unchanged
// items aren't touched.
SyncSourceReport::ItemState state =
it == revisions.end() ?
SyncSourceReport::ITEM_ADDED : // not found in database, create anew
SyncSourceReport::ITEM_UPDATED; // found, update existing item
try {
report.incrementItemStat(report.ITEM_LOCAL,
state,
report.ITEM_TOTAL);
if (!dryrun) {
insertItem(it == revisions.end() ? "" : uid,
item);
}
} catch (...) {
report.incrementItemStat(report.ITEM_LOCAL,
state,
report.ITEM_REJECT);
throw;
}
}
// remove handled item from revision list so
// that when we are done, the only remaining
// items listed there are the ones which did
// no exist in the backup
if (it != revisions.end()) {
revisions.erase(it);
}
}
// now remove items that were not in the backup
BOOST_FOREACH(const StringPair &mapping, revisions) {
try {
report.incrementItemStat(report.ITEM_LOCAL,
report.ITEM_REMOVED,
report.ITEM_TOTAL);
if (!dryrun) {
deleteItem(mapping.first);
}
} catch(...) {
report.incrementItemStat(report.ITEM_LOCAL,
report.ITEM_REMOVED,
report.ITEM_REJECT);
throw;
}
}
}

View file

@ -75,9 +75,10 @@ class TrackingSyncSource : public EvolutionSyncSource
virtual void backupData(const string &dirname, ConfigNode &node, BackupReport &report);
/**
* Restore database from data stored in backupData().
* Restore database from data stored in backupData(). Will be
* called inside open()/close() pair. beginSync() is *not* called.
*/
virtual void restoreData(const string &dirname, const ConfigNode &node);
virtual void restoreData(const string &dirname, const ConfigNode &node, bool dryrun, SyncSourceReport &report);
typedef map<string, string> RevisionMap_t;