test: convert scripts to Python3

This is the result of 2to3, with the interpreter path updated manually,
tabs replaced by spaces and some obsolete packages removed.

Required because the current nightly build host no longer has Python2 installed.

Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
This commit is contained in:
Patrick Ohly 2020-07-24 13:36:00 -07:00
parent 75dff12823
commit edf1314def
7 changed files with 583 additions and 650 deletions

View File

@ -43,11 +43,13 @@ nodist_bin_SCRIPTS += src/syncevo-http-server
endif
src/syncevo-http-server: $(top_srcdir)/test/syncevo-http-server.py
$(AM_V_GEN)sed -e 's|\@PYTHON\@|$(PYTHON)|' $< > $@
$(AM_V_GEN)chmod a+x $@
CLEANFILES += src/syncevo-http-server
nodist_bin_SCRIPTS += src/syncevo-phone-config
src/syncevo-phone-config: $(top_srcdir)/test/syncevo-phone-config.py
$(AM_V_GEN)sed -e 's|\@PYTHON\@|$(PYTHON)|' $< > $@
$(AM_V_GEN)chmod a+x $@
CLEANFILES += src/syncevo-phone-config
SYNCEVOLUTION_DEP =

View File

@ -1,4 +1,4 @@
#!@PYTHON@
#!/usr/bin/python3
"""
Converts the .log output for a client-test test into HTML, with
@ -13,7 +13,7 @@ filename = sys.argv[1]
if filename == '-':
log = sys.stdin
else:
log = open(filename)
log = open(filename, encoding='utf-8')
out = sys.stdout

View File

@ -1,4 +1,4 @@
#! /usr/bin/python
#! /usr/bin/python3
#
# Copyright (C) 2014 Intel Corporation
#

View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/python3
'''
Copyright (C) 2009 Intel Corporation
@ -41,12 +41,12 @@ def check (resultdir, serverlist,resulturi, srcdir, shellprefix, backenddir):
servers = serverlist.split(",")
else:
servers = []
result = open("%s/nightly.xml" % resultdir,"w")
result = open("%s/nightly.xml" % resultdir,"w",encoding='utf-8')
result.write('''<?xml version="1.0" encoding="utf-8" ?>\n''')
result.write('''<nightly-test>\n''')
indents=[space]
if(os.path.isfile(resultdir+"/output.txt")==False):
print "main test output file not exist!"
print("main test output file not exist!")
else:
indents,cont = step1(resultdir,result,indents,resultdir,resulturi, shellprefix, srcdir)
if (cont):
@ -62,7 +62,7 @@ patchsummary = re.compile('^Subject: (?:\[PATCH.*?\] )?(.*)\n')
patchauthor = re.compile('^From: (.*?) <.*>\n')
def extractPatchSummary(patchfile):
author = ""
for line in open(patchfile):
for line in open(patchfile, encoding='utf-8'):
m = patchauthor.match(line)
if m:
author = m.group(1) + " - "
@ -92,7 +92,7 @@ def step1(resultdir, result, indents, dir, resulturi, shellprefix, srcdir):
if m:
name = m.group(1)
result.write(' <source name="%s"><description><![CDATA[%s]]></description>\n' %
(name, open(os.path.join(resultdir, source)).read()))
(name, open(os.path.join(resultdir, source), encoding='utf-8').read()))
result.write(' <patches>\n')
for patch in files:
if fnmatch.fnmatch(patch, name + '-*.patch'):
@ -106,16 +106,16 @@ def step1(resultdir, result, indents, dir, resulturi, shellprefix, srcdir):
indent =indents[-1]+space
indents.append(indent)
result.write(indent+'''<cpuinfo>\n''')
fout=subprocess.check_output('cat /proc/cpuinfo|grep "model name" |uniq', shell=True)
fout=subprocess.check_output('cat /proc/cpuinfo|grep "model name" |uniq', shell=True).decode('utf-8')
result.write(indent+fout)
result.write(indent+'''</cpuinfo>\n''')
result.write(indent+'''<memoryinfo>\n''')
fout=subprocess.check_output('cat /proc/meminfo|grep "Mem"', shell=True)
fout=subprocess.check_output('cat /proc/meminfo|grep "Mem"', shell=True).decode('utf-8')
for s in fout.split('\n'):
result.write(indent+s)
result.write(indent+'''</memoryinfo>\n''')
result.write(indent+'''<osinfo>\n''')
fout=subprocess.check_output('uname -osr'.split())
fout=subprocess.check_output('uname -osr'.split()).decode('utf-8')
result.write(indent+fout)
result.write(indent+'''</osinfo>\n''')
if 'schroot' in shellprefix:
@ -123,9 +123,9 @@ def step1(resultdir, result, indents, dir, resulturi, shellprefix, srcdir):
# Don't make assumption about the schroot invocation. Instead
# extract the schroot name from the environment of the shell.
name=subprocess.check_output(shellprefix + "sh -c 'echo $SCHROOT_CHROOT_NAME'",
shell=True)
shell=True).decode('utf-8')
info = re.sub(r'schroot .*', 'schroot -i -c ' + name, shellprefix)
fout=subprocess.check_output(info, shell=True)
fout=subprocess.check_output(info, shell=True).decode('utf-8')
s = []
for line in fout.split('\n'):
m = re.match(r'^\s+(Name|Description)\s+(.*)', line)
@ -139,7 +139,7 @@ def step1(resultdir, result, indents, dir, resulturi, shellprefix, srcdir):
for lib in libs:
try:
fout=subprocess.check_output(shellprefix+' pkg-config --modversion '+lib +' |grep -v pkg-config',
shell=True)
shell=True).decode('utf-8')
s = s + lib +': '+fout +' '
except subprocess.CalledProcessError:
pass
@ -160,10 +160,10 @@ def step1(resultdir, result, indents, dir, resulturi, shellprefix, srcdir):
'dist':'dist'}
for tag in tags:
result.write(indent+'''<'''+tagsp[tag])
fout=subprocess.check_output('find `dirname '+input+'` -type d -name *'+tag, shell=True)
fout=subprocess.check_output('find `dirname '+input+'` -type d -name *'+tag, shell=True).decode('utf-8')
s = fout.rpartition('/')[2].rpartition('\n')[0]
result.write(' path ="'+s+'">')
'''check the result'''
'''check the result'''
if 0 == os.system("grep -q '^"+tag+": .*: failed' "+input):
result.write("failed")
elif 0 == os.system ("grep -q '^"+tag+" successful' "+input):
@ -198,9 +198,9 @@ def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
cmd='sed -n '
for server in servers:
cmd+= '-e /^'+server+'/p '
print "Analyzing overall result %s" % (resultdir+'/output.txt')
print("Analyzing overall result %s" % (resultdir+'/output.txt'))
cmd = cmd +resultdir+'/output.txt'
fout=subprocess.check_output(cmd, shell=True)
fout=subprocess.check_output(cmd, shell=True).decode('utf-8')
for line in fout.split('\n'):
for server in servers:
# Find first line with "foobar successful" or "foobar: <command failure>",
@ -211,7 +211,7 @@ def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
t=t.partition(':')[2]
t = t.strip()
if t != 'skipped: disabled in configuration':
print "Result for %s: %s" % (server, t)
print("Result for %s: %s" % (server, t))
params[server]=t
indent =indents[-1]+space
@ -267,7 +267,7 @@ def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
haveSync = False
for server in servers:
matched = False
'''Only process servers listed in the input parametr'''
'''Only process servers listed in the input parametr'''
for rserver in runservers:
if(rserver.find('-')!= -1 and rserver.partition('-')[2] == server):
matched = True
@ -292,7 +292,7 @@ def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
clientSync = re.compile(r' +Client::Sync::(.*?)::(?:(Suspend|Resend|Retry)::)?([^:]+)')
for source in ('file_task', 'file_event', 'file_contact', 'eds_contact', 'eds_event'):
cmd = shellprefix + " env LD_LIBRARY_PATH=%s/build-synthesis/src/.libs SYNCEVOLUTION_BACKEND_DIR=%s CLIENT_TEST_PEER_CAN_RESTART=1 CLIENT_TEST_RETRY=t CLIENT_TEST_RESEND=t CLIENT_TEST_SUSPEND=t CLIENT_TEST_SOURCES=%s %s/client-test -h" % (srcdir, backenddir, source, srcdir)
fout=subprocess.check_output(cmd, shell=True)
fout=subprocess.check_output(cmd, shell=True).decode('utf-8')
for line in fout.split('\n'):
m = clientSync.match(line)
if m:
@ -323,10 +323,9 @@ def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
result.write('result="%s"' % m.group(1))
result.write('>\n')
# sort files by creation time, to preserve run order
logs = map(lambda file: (os.stat(file).st_mtime, file),
glob.glob(resultdir+'/'+rserver+'/*.txt'))
logs = [(os.stat(file).st_mtime, file) for file in glob.glob(resultdir+'/'+rserver+'/*.txt')]
logs.sort()
logs = map(lambda entry: entry[1], logs)
logs = [entry[1] for entry in logs]
logdic ={}
logprefix ={}
if server in ('dbus', 'pim'):
@ -364,7 +363,7 @@ def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
logfile = None
htmlfile = None
linetype = None
for line in open(rserver + "/output.txt"):
for line in open(rserver + "/output.txt", encoding='utf-8'):
m = test_start.search(line)
if m:
is_okay = True
@ -378,12 +377,12 @@ def step2(resultdir, result, servers, indents, srcdir, shellprefix, backenddir):
newname = rserver + "/" + cl + "_" + func + ".txt"
if newname != name:
name = newname
logfile = open(name, "w")
logfile = open(name, "w", encoding='utf-8')
if htmlfile:
htmlfile.write('</pre></body')
htmlfile.close()
htmlfile = None
htmlfile = open(name + ".html", "w")
htmlfile = open(name + ".html", "w", encoding='utf-8')
htmlfile.write('''<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
@ -465,7 +464,7 @@ span.hl { color: #c02020 }
# <path>/N7SyncEvo11CmdlineTestE_testConfigure.txt - C++ name mangling?
m = re.match(r'^(Client_Source_|Client_Sync_|N7SyncEvo\d+|[^_]*_)(.*)_([^_]*)\.txt', logname)
if not m or logname.endswith('.server.txt'):
print "skipping", logname
print("skipping", logname)
continue
# Client_Sync_, Client_Source_, SyncEvo_, ...
prefix = m.group(1)
@ -482,7 +481,7 @@ span.hl { color: #c02020 }
casename = m.group(2) + '::' + casename
if '.' in casename:
# Ignore sub logs.
print "skipping", logname
print("skipping", logname)
continue
# Another special case: suspend/resend/retry uses an intermediate grouping
# which we can ignore because the name is repeated in the test case name.
@ -495,12 +494,12 @@ span.hl { color: #c02020 }
# # Ignore sub logs.
# print "skipping", logname
# continue
print "analyzing log %s: prefix %s, subset %s, testcase %s" % (logname, prefix, format, casename)
print("analyzing log %s: prefix %s, subset %s, testcase %s" % (logname, prefix, format, casename))
if(format not in logdic):
logdic[format]=[]
logdic[format].append((casename, log))
logprefix[format]=prefix
for format in logdic.keys():
for format in list(logdic.keys()):
indent +=space
indents.append(indent)
prefix = logprefix[format]
@ -554,6 +553,6 @@ if(__name__ == "__main__"):
if (len(sys.argv)!=7):
# srcdir and basedir must be usable inside the shell started by shellprefix (typically
# the chroot).
print "usage: python resultchecker.py resultdir servers resulturi srcdir shellprefix backenddir"
print("usage: python resultchecker.py resultdir servers resulturi srcdir shellprefix backenddir")
else:
check(*sys.argv[1:])

View File

@ -1,4 +1,4 @@
#!/usr/bin/python -u
#!/usr/bin/python3 -u
"""
The general idea is that tests to run are defined as a list of
@ -13,7 +13,7 @@ the result of each action:
that the action can put there
"""
import os, sys, popen2, traceback, re, time, smtplib, optparse, stat, shutil, StringIO, MimeWriter
import os, sys, traceback, re, time, smtplib, optparse, stat, shutil, io
import shlex
import subprocess
import fnmatch
@ -21,11 +21,25 @@ import copy
import errno
import signal
import stat
import exceptions
import email.message
import email.headerregistry
import pathlib
class Unbuffered(object):
def __init__(self, stream):
self.stream = stream
def write(self, data):
self.stream.write(data)
self.stream.flush()
def writelines(self, datas):
self.stream.writelines(datas)
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
def log(format, *args):
now = time.time()
print 'runtests.py-%d' % os.getpid(), time.asctime(time.gmtime(now)), 'UTC', '(+ %.1fs / %.1fs)' % (now - log.latest, now - log.start), format % args
print('runtests.py-%d' % os.getpid(), time.asctime(time.gmtime(now)), 'UTC', '(+ %.1fs / %.1fs)' % (now - log.latest, now - log.start), format % args)
log.latest = now
log.start = time.time()
log.latest = log.start
@ -77,7 +91,7 @@ def del_dir(path):
# We might have skipped deleting something, allow that.
try:
os.rmdir(path)
except OSError, ex:
except OSError as ex:
if ex.errno != errno.ENOTEMPTY:
raise
@ -108,9 +122,9 @@ def copyLog(filename, dirname, htaccess, lineFilter=None):
outname = outname + ".gz"
out = gzip.open(outname, "wb")
else:
out = file(outname, "w")
out = open(outname, "w", encoding="utf-8")
error = None
for line in file(filename, "r").readlines():
for line in open(filename, "r", encoding="utf-8").readlines():
if not error and line.find("ERROR") >= 0:
error = line
if lineFilter:
@ -128,7 +142,7 @@ def copyLog(filename, dirname, htaccess, lineFilter=None):
def TryKill(pid, signal):
try:
os.kill(pid, signal)
except OSError, ex:
except OSError as ex:
# might have quit in the meantime, deal with the race
# condition
if ex.errno != 3:
@ -205,7 +219,7 @@ class Jobserver:
def _unblock(self):
'''Unblock signals if blocked and we currently own no slots.'''
if self.blocked and not self.allocated:
for sig, handler in self.blocked.items():
for sig, handler in list(self.blocked.items()):
signal.signal(sig, handler)
self.blocked = {}
@ -269,7 +283,7 @@ class Action:
fd = os.open("output.txt", os.O_WRONLY|os.O_CREAT|os.O_APPEND)
os.dup2(fd, 1)
os.dup2(fd, 2)
sys.stdout = os.fdopen(fd, "w", 0) # unbuffered output!
sys.stdout = Unbuffered(os.fdopen(fd, "w"))
sys.stderr = sys.stdout
if self.needhome and context.home_template:
# Clone home directory template?
@ -292,7 +306,7 @@ class Action:
if self.needhome and not context.home_template:
self.wait_for_completion()
except Exception, inst:
except Exception as inst:
# fork() error handling in parent.
traceback.print_exc()
self.status = Action.FAILED
@ -320,7 +334,7 @@ class Context:
def __init__(self, tmpdir, resultdir, uri, workdir, mailtitle, sender, recipients, mailhost, enabled, skip, nologs, setupcmd, make, sanitychecks, lastresultdir, datadir):
# preserve normal stdout because stdout/stderr will be redirected
self.out = os.fdopen(os.dup(1), "w", 0) # unbuffered
self.out = Unbuffered(os.fdopen(os.dup(1), "w"))
self.todo = []
self.actions = {}
self.tmpdir = abspath(tmpdir)
@ -380,7 +394,7 @@ class Context:
cmd.insert(0, 'env')
if not runAsIs:
cmdstr = " ".join(map(lambda x: (' ' in x or '(' in x or '\\' in x or x == '') and ("'" in x and '"%s"' or "'%s'") % x or x, cmd))
cmdstr = " ".join([(' ' in x or '(' in x or '\\' in x or x == '') and ("'" in x and '"%s"' or "'%s'") % x or x for x in cmd])
if dumpCommands:
cmdstr = "set -x; " + cmdstr
@ -492,7 +506,7 @@ class Context:
run_servers.append(action.name);
action.tryexecution(step, not self.nologs)
started.append(action)
except Exception, inst:
except Exception as inst:
traceback.print_exc()
self.summary.append("%s failed: %s" % (action.name, inst))
@ -511,7 +525,7 @@ class Context:
s.close()
# copy information about sources
for source in self.actions.keys():
for source in list(self.actions.keys()):
action = self.actions[source]
basedir = getattr(action, 'basedir', None)
if basedir and os.path.isdir(basedir):
@ -554,20 +568,18 @@ class Context:
self.runCommand(" && ".join(commands), dumpCommands=True)
# report result by email
server, body, writer = self.startEmail()
if server:
msg=''
msg = self.startEmail()
if msg:
try:
msg = open(self.resultdir + "/nightly.html").read()
html_body = pathlib.Path(self.resultdir + "/nightly.html", encoding="utf-8").read_text()
except IOError:
msg = '''<html><body><h1>Error: No HTML report generated!</h1></body></html>\n'''
html_body = '''<html><body><h1>Error: No HTML report generated!</h1></body></html>\n'''
# insert absolute URL into hrefs so that links can be opened directly in
# the mail reader
msg = re.sub(r'href="([a-zA-Z0-9./])',
'href="' + uri + r'/\1',
msg)
writer.startbody("text/html;charset=ISO-8859-1").write(msg)
self.finishEmail(server, body)
html_body = re.sub(r'href="([a-zA-Z0-9./])',
'href="' + uri + r'/\1',
html_body)
self.finishEmail(msg, html_body)
else:
log('%s\n', '\n'.join(self.summary))
@ -577,25 +589,21 @@ class Context:
sys.exit(1)
def startEmail(self):
# TODO: enable sending of mails again, using
# email package instead of MimeWriter.
if self.recipients:
server = smtplib.SMTP(self.mailhost)
body = StringIO.StringIO()
writer = MimeWriter.MimeWriter (body)
writer.addheader("From", self.sender)
for recipient in self.recipients:
writer.addheader("To", recipient)
writer.addheader("Subject", self.mailtitle + ": " + os.path.basename(self.resultdir))
writer.addheader("MIME-Version", "1.0")
writer.flushheaders()
return (server, body, writer)
msg = email.message.EmailMessage()
msg['Subject'] = self.mailtitle + ": " + os.path.basename(self.resultdir)
msg['From'] = email.headerregistry.Address(addr_spec=self.sender)
msg['To'] = [email.headerregistry.Address(addr_spec=x) for x in self.recipients]
return msg
else:
return (None, None, None)
return None
def finishEmail(self, server, body):
failed = server.sendmail(self.sender, self.recipients, body.getvalue())
if failed:
log('could not send to: %s', failed)
sys.exit(1)
def finishEmail(self, msg, html_body):
msg.set_content(html_body, subtype='html')
with smtplib.SMTP(self.mailhost) as s:
s.send_message(msg)
class CVSCheckout(Action):
"""Does a CVS checkout (if directory does not exist yet) or an update (if it does)."""
@ -750,7 +758,7 @@ class GitCopy(GitCheckoutBase, Action):
'(cd ..; for i in [0-9]*.patch; do [ ! -f "$i" ] || mv $i %(name)s-$i; done)',
'git describe --tags --always nightly | sed -e "s/\(.*\)-\([0-9][0-9]*\)-g\(.*\)/\\1 + \\2 commit(s) = \\3/" >>%(patchlog)s',
'( git status | grep -q "working directory clean" && echo "working directory clean" || ( echo "working directory dirty" && ( echo From: nightly testing ; echo Subject: [PATCH 1/1] uncommitted changes ; echo ; git status; echo; git diff HEAD ) >../%(name)s-1000-unstaged.patch ) ) >>%(patchlog)s'
]) % self
]) % self.__dict__
context.runCommand(cmd, dumpCommands=True, runAsIs=True, jobs=None)
if os.access("autogen.sh", os.F_OK):
@ -919,7 +927,7 @@ class SyncEvolutionTest(Action):
finally:
tocopy = re.compile(r'.*\.txt|.*\.log|.*\.client.[AB]|.*\.(cpp|h|c)\.html|.*\.txt\.html|.*\.log.html')
toconvert = re.compile(r'Client_.*\.txt')
htaccess = file(os.path.join(resdir, ".htaccess"), "a")
htaccess = open(os.path.join(resdir, ".htaccess"), "a", encoding="utf-8")
for f in os.listdir(actiondir):
if tocopy.match(f):
error = copyLog(f, resdir, htaccess, self.lineFilter)
@ -1093,7 +1101,7 @@ class EvoSvn(Action):
makeoptions contain additional parameters for make (like BRANCH=2.20 PREFIX=/tmp/runtests/evo)."""
Action.__init__(self,name)
self.workdir = workdir
self.resultdir = resultdir
self.resultdir = resultdir
self.makedir = makedir
self.makeoptions = makeoptions
@ -1104,9 +1112,9 @@ class EvoSvn(Action):
if os.access(self.resultdir, os.F_OK):
shutil.rmtree(self.resultdir)
os.system("rm -f .stamp/*.install")
localmk = open("local.mk", "a")
localmk.write("PREFIX := %s\n" % self.resultdir)
localmk.close()
localmk = open("local.mk", "a")
localmk.write("PREFIX := %s\n" % self.resultdir)
localmk.close()
if os.access(".stamp", os.F_OK):
context.runCommand("make check-changelog")
context.runCommand("%s %s" % (context.make, self.makeoptions))
@ -1115,7 +1123,7 @@ for evosvn in options.evosvn:
name, path = evosvn.split("=")
evosvn = EvoSvn("evolution" + name,
os.path.join(options.tmpdir, "evolution%s-build" % name),
os.path.join(options.tmpdir, "evolution%s-result" % name),
os.path.join(options.tmpdir, "evolution%s-result" % name),
path,
"SUDO=true")
context.add(evosvn)
@ -1553,7 +1561,7 @@ localtests.append(test)
context.add(test)
# Implement the mapping from "evolution" to the new test names.
if enabled.has_key("evolution"):
if "evolution" in enabled:
if enabled["evolution"] is None:
# Everything is enabled.
for test in localtests:
@ -1571,7 +1579,7 @@ if enabled.has_key("evolution"):
if defTest.startswith(e):
localtestsEnabled.setdefault(localtest.name, []).append(e)
break
for name, e in localtestsEnabled.iteritems():
for name, e in localtestsEnabled.items():
enabled[name] = ','.join(e)
# test-dbus.py itself doesn't need to run under valgrind, remove it...
@ -2511,21 +2519,21 @@ context.add(ovitest)
if options.list:
for action in context.todo:
print action.name
print(action.name)
else:
pid = os.getpid()
log('Ready to run. I have PID %d.', pid)
try:
context.execute()
except exceptions.SystemExit:
except SystemExit:
raise
except:
# Something went wrong. Send emergency email if an email is
# expected and we are the parent process.
if pid == os.getpid():
server, body, writer = context.startEmail()
if server:
writer.startbody("text/html;charset=ISO-8859-1").write('<html><body><pre>%s</pre></body></html>' %
msg = context.startEmail()
if msg:
context.finishEmail(msg,
'<html><body><pre>%s</pre></body></html>' %
traceback.format_exc())
context.finishEmail(server, body)
raise

View File

@ -5,9 +5,9 @@ Runs a SyncML HTTP server under the given base URL.'''
# use the same glib main loop in D-Bus and twisted
from dbus.mainloop.glib import DBusGMainLoop
from twisted.internet import glib2reactor # for non-GUI apps
from twisted.internet import gireactor # for non-GUI apps
DBusGMainLoop(set_as_default=True)
glib2reactor.install()
gireactor.install()
import dbus
import sys

File diff suppressed because it is too large Load Diff