development/webtester
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new commands testsuite and testcase

Browse Source
This commit is contained in:
Marc Wäckerlin
2015-05-09 12:27:03 +00:00
parent ee1ebd8ca9
commit 9335d5f6bb
8 changed files with 329 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
src/commands.hxx
View File

@@ -462,35 +462,45 @@ class Script: public QObject {
linenr+=oldsize-in.size())
_script.push_back(parse(in, linenr));
}
void run(QWebFrame* frame, QString td = QString(), bool screenshots = true,
int maxretries = 0) {
assert(_internalTestsuiteNode);
run(frame, *_internalTestsuiteNode, td,
screenshots, maxretries); //( @todo extract from parent
QStringList print() {
QStringList result;
for (auto cmd(_script.begin()); cmd!=_script.end(); ++cmd) {
result += (*cmd)->command();
}
return result;
}
void run(QWebFrame* frame) {
run(frame, _testsuites, targetdir(), _screenshots, _maxretries);
}
void run(QWebFrame* frame, xml::Node& testsuiteNode,
void run(QWebFrame* frame, std::shared_ptr<xml::Node> testsuites,
QString td = QString(), bool screenshots = true,
int maxretries = 0) {
_internalTestsuiteNode = &testsuiteNode;
assert(_internalTestsuiteNode);
_testsuites = testsuites;
_timeout = 20; // defaults to 20s
_ignoreSignalsUntil.clear();
addSignals(frame);
_screenshots = screenshots;
_maxretries = maxretries;
_timer.setSingleShot(true);
targetdir(!td.isEmpty()
? td
: _testsuites->children()
? xmlstr(_testsuites->last().attr("name"))
: "attachments");
if (!_testsuites->children()) {
xml::Node testsuite("testsuite");
testsuite.attr("name") = "Unnamed Test Suite";
(*_testsuites)<<testsuite;
}
int retries(0), back(0);
for (auto cmd(_script.begin()); cmd!=_script.end(); ++cmd) {
xml::Node testcase("testcase");
try {
testcase.attr("classname") =
testsuiteNode.attr("name");
//xmlattr((*cmd)->command(), true).toStdString();
testcase.attr("classname") = xmlattr(_testclass, true).toStdString();
testcase.attr("name") =
xmlattr((*cmd)->tag(), true).toStdString();
xmlattr((*cmd)->command(), true).toStdString();
if (!_ignores.size() || (*cmd)->tag()=="label") { // not ignored
_timer.start(_timeout*1000);
testsuite(xmlstr(testsuiteNode.attr("name")));
targetdir(!td.isEmpty() ? td : xmlstr(testsuiteNode.attr("name")));
try {
if (!(*cmd)->execute(this, frame)) {
_timer.stop();
@@ -501,7 +511,7 @@ class Script: public QObject {
xmlattr(_cerr).toStdString());
_cout.clear();
_cerr.clear();
testsuiteNode<<testcase;
_testsuites->last()<<testcase;
break; // test is successfully finished
}
} catch (PossibleRetryLoad& e) {
@@ -511,7 +521,7 @@ class Script: public QObject {
try { // take a screenshot on error
QString filename(Screenshot::screenshot
((*cmd)->line(), targetdir(),
QFileInfo(testsuite()).baseName(),
_testclass,
QString("retry-%1")
.arg((ulong)retries, 2, 10,
QLatin1Char('0')),
@@ -563,7 +573,7 @@ class Script: public QObject {
xmlattr(_cerr).toStdString());
_cout.clear();
_cerr.clear();
testsuiteNode<<testcase;
_testsuites->last()<<testcase;
}
} catch (Exception& e) {
_timer.stop();
@@ -574,7 +584,7 @@ class Script: public QObject {
xmlattr(_cerr).toStdString());
_cout.clear();
_cerr.clear();
testsuiteNode<<testcase;
_testsuites->last()<<testcase;
removeSignals(frame);
e.line((*cmd)->line());
if (screenshots)
@@ -582,22 +592,20 @@ class Script: public QObject {
{
QString filename(Screenshot::sourceHtml
((*cmd)->line(), targetdir(),
QFileInfo(testsuite()).baseName(),
_testclass,
"error", frame));
plainlog("[[ATTACHMENT|"+filename+"]]");
} {
QString filename(Screenshot::screenshot
((*cmd)->line(), targetdir(),
QFileInfo(testsuite()).baseName(),
_testclass,
"error", frame));
plainlog("[[ATTACHMENT|"+filename+"]]");
}
} catch (... ) {} // ignore exception in screenshot
_internalTestsuiteNode = 0;
throw;
}
}
_internalTestsuiteNode = 0;
removeSignals(frame);
if (!_signals.empty()) throw UnhandledSignals(_signals);
}
@@ -613,18 +621,26 @@ class Script: public QObject {
bool screenshots() {
return _screenshots;
}
void testsuite(QString name) {
_testsuite = name;
}
QString testsuite() {
return _testsuite;
}
void targetdir(QString name) {
_targetdir = name;
}
QString targetdir() {
return _targetdir;
}
void testclass(QString tc) {
_testclass = tc;
}
QString testclass() {
return _testclass;
}
void testsuite(QString name) {
xml::Node testsuite("testsuite");
testsuite.attr("name") = xmlattr(name, true).toStdString();
testsuite.attr("timestamp") =
QDateTime::currentDateTime().toString(Qt::ISODate).toStdString();
_testsuites->last().attr("failures") = "0";
(*_testsuites)<<testsuite;
}
Signal getSignal() {
while (!_signals.size()) QCoreApplication::processEvents();
Signal res(_signals.front());
@@ -655,10 +671,11 @@ class Script: public QObject {
_rvariables = o._rvariables;
_timeout = o._timeout;
_clicktype = o._clicktype;
_testsuite = o._testsuite;
_internalTestsuiteNode = o._internalTestsuiteNode;
assert(_internalTestsuiteNode);
_testsuites = o._testsuites;
_testclass = o._testclass;
_targetdir = o._targetdir;
_maxretries = o._maxretries;
_screenshots = o._screenshots;
_cout.clear();
_cerr.clear();
_ignoreSignalsUntil.clear();
@@ -835,15 +852,16 @@ class Script: public QObject {
QString _cout;
QString _cerr;
bool _screenshots;
int _maxretries;
QString _ignoreSignalsUntil;
QMap<QString, QString> _variables; ///< variable mapping
QMap<LenString, LenString> _rvariables; ///< reverse variable mapping
QMap<QString, std::shared_ptr<Function> > _functions;
int _timeout;
ClickType _clicktype;
QString _testsuite;
QString _targetdir;
xml::Node* _internalTestsuiteNode; ///< only valid within run
std::shared_ptr<xml::Node> _testsuites; ///< only valid within run
QString _testclass;
};
class Do: public Command {
@@ -1892,7 +1910,8 @@ class If: public Command {
"Match allows a regular expression.";
}
QString command() const {
return tag();
return tag()+" "+_variable+" "+QString(_cmp)+" "+_value
+(_script.get()?"\n"+_script->print().join("\n "):"");
}
std::shared_ptr<Command> parse(Script*, QString args,
QStringList& in, int) {
@@ -1934,6 +1953,64 @@ class If: public Command {
std::shared_ptr<Script> _script;
};
class TestSuite: public Command {
public:
QString tag() const {
return "testsuite";
}
QString description() const {
return
tag()+" <name>"
"\n\n"
"Start a testsuite and give it a name.";
}
QString command() const {
return tag()+" "+_name;
}
std::shared_ptr<Command> parse(Script*, QString args,
QStringList&, int) {
std::shared_ptr<TestSuite> cmd(new TestSuite());
cmd->_name = args;
return cmd;
}
bool execute(Script* script, QWebFrame* frame) {
Logger log(this, script);
script->testsuite(script->replacevars(_name));
return true;
}
private:
QString _name;
};
class TestCase: public Command {
public:
QString tag() const {
return "testcase";
}
QString description() const {
return
tag()+" <name>"
"\n\n"
"Start a testcase and give it a name.";
}
QString command() const {
return tag()+" "+_name;
}
std::shared_ptr<Command> parse(Script*, QString args,
QStringList&, int) {
std::shared_ptr<TestCase> cmd(new TestCase());
cmd->_name = args;
return cmd;
}
bool execute(Script* script, QWebFrame* frame) {
Logger log(this, script);
script->testclass(script->replacevars(_name));
return true;
}
private:
QString _name;
};
/* Template:
class : public Command {
public:
@@ -1965,7 +2042,7 @@ inline bool Screenshot::execute(Script* script, QWebFrame* frame) {
if (!script->screenshots()) return true;
Logger log(this, script);
QString filename(screenshot(line(), script->targetdir(),
QFileInfo(script->testsuite()).baseName(),
script->testclass(),
script->replacevars(_filename), frame));
log["[[ATTACHMENT|"+filename+"]]"];
return true;
@@ -2004,7 +2081,7 @@ inline void Command::subScript(std::shared_ptr<Script> script,
connect(script.get(), SIGNAL(logging(QString)),
parent, SLOT(log(QString)));
parent->removeSignals(frame);
script->run(frame, parent->targetdir());
script->run(frame);
parent->addSignals(frame);
disconnect(script.get(), SIGNAL(logging(QString)),
parent, SLOT(log(QString)));
@@ -2042,6 +2119,8 @@ inline void Script::initPrototypes() {
add(new Function);
add(new Call);
add(new If);
add(new TestSuite);
add(new TestCase);
}
#endif

21
src/testgui.hxx
View File

@@ -117,23 +117,17 @@ class TestGUI: public QMainWindow, protected Ui::TestGUI {
try {
Script script;
connect(&script, SIGNAL(logging(QString)), SLOT(logging(QString)));
xml::Node testsuite("testsuite");
std::shared_ptr<xml::Node> testsuites(new xml::Node("testsuite"));
if (_setupscriptactive->isEnabled()
&& _setupscriptactive->isChecked()) {
script.parse(_setupscript->toPlainText().split('\n'));
testsuite.attr("name") = "setup-script";
testsuite.attr("timestamp") =
QDateTime::currentDateTime().toString(Qt::ISODate).toStdString();
script.run(_web->page()->mainFrame(), testsuite, QString(), false);
script.run(_web->page()->mainFrame(), testsuites, QString(), false);
script.reset();
}
QString text(_testscript->textCursor().selection().toPlainText());
if (text.isEmpty()) text = _testscript->toPlainText();
script.parse(text.split('\n'));
testsuite.attr("name") = "setup-script";
testsuite.attr("timestamp") =
QDateTime::currentDateTime().toString(Qt::ISODate).toStdString();
script.run(_web->page()->mainFrame(), testsuite, QString(), false);
script.run(_web->page()->mainFrame(), testsuites, QString(), false);
} catch (std::exception &x) {
QMessageBox::critical(this, tr("Script Failed"),
tr("Script failed with message:\n%1")
@@ -222,17 +216,14 @@ class TestGUI: public QMainWindow, protected Ui::TestGUI {
_setupscriptactive->setEnabled(false);
try {
_setupscriptstatus->setText(trUtf8("?"));
xml::Node testsuite("testsuite");
testsuite.attr("name") = "setup-script";
testsuite.attr("timestamp") =
QDateTime::currentDateTime().toString(Qt::ISODate).toStdString();
std::shared_ptr<xml::Node> testsuites(new xml::Node("testsuite"));
Script script;
TestWebPage page(0, true);
script.parse(_setupscript->toPlainText().split('\n'));
script.run(page.mainFrame(), testsuite, QString(), false);
script.run(page.mainFrame(), testsuites, QString(), false);
_setupScript.cleanup();
_setupScript.parse(_setupscript->toPlainText().split('\n'));
_setupScript.run(page.mainFrame(), testsuite, QString(), false);;
_setupScript.run(page.mainFrame(), testsuites, QString(), false);;
_setupscriptstatus->setText(trUtf8("✔"));
_setupscriptactive->setEnabled(true);
} catch (std::exception &x) {

71
src/webrunner.cxx
View File

@@ -16,6 +16,42 @@
#include <version.hxx>
using namespace NAMESPACE;
/** @page junitXml Test Output XML Format
The test output XML format emulates JUnit's format, so that JUnit
tools can be used to evaluate ist, namely on tools like jenkins.
@code{.xml}
<testsuites>
<testsuite name="..." timestamp="..." failures="...">
<testcase classname="..." name="...">
<failure message="..." />
<system-out>...</system-out>
<system-err>...</system-err>
</testcase>
<testcase ... />
</testsuite>
<testsuite ...>
...
</testsuite>
<testsuites>
@endcode
The default name for a testsuite is the name of the test script
file. it can be overwritten using the @c testsuite statement. The
attribute @c failures contains the number of failed test cases.
The default classname for a testcase is @c testsuite-preparation
before the script runs, then in the script it is the name of the
current command and the classname equals to the testsuite's name.
If you specify testsuites, testcases in your script file, then
this overwrites the defaults. Here @c testsuite maps to the
testsuite's name, @c testcase maps to the next testcase's
classname.
*/
QString format(QString txt, int indent = 2, int cpl = 60) {
QStringList res;
QStringList lines(txt.split('\n'));
@@ -102,7 +138,7 @@ int main(int argc, char *argv[]) try {
int height(parser.value("height").toInt());
QString target(parser.value("target-path"));
p.resize(width, height);
xml::Node testsuites("testsuites");
std::shared_ptr<xml::Node> testsuites(new xml::Node("testsuites"));
Q_FOREACH(QString file, parser.positionalArguments()) {
int expectedtestcases(-1);
xml::Node testsuite("testsuite");
@@ -142,47 +178,48 @@ int main(int argc, char *argv[]) try {
testsuite<<testcase;
}
} else {
script.run(p.page()->mainFrame(), testsuite,
testsuite.attr("failures") = "0";
(*testsuites)<<testsuite;
script.run(p.page()->mainFrame(), testsuites,
target+QDir::separator()+QFileInfo(file).baseName(),
true, retries);
testsuite.attr("failures") = "0";
script.log("SUCCESS: "+file);
}
} catch (std::exception& e) {
script.log("FAILED: "+file+" with "+e.what());
xml::Node failure("failure");
failure.attr("message") = Script::xmlattr(e.what()).toStdString();
testsuite[testsuite.children()-1]<<failure;
testsuites->last().last()<<failure;
if (expectedtestcases==-1)
testsuite.attr("failures") = "1";
testsuites->last().attr("failures") = "1";
else
testsuite.attr("failures") =
mrw::string(expectedtestcases+1-testsuite.children());
testsuites->last().attr("failures") =
mrw::string(expectedtestcases+1-testsuites->last().children());
if (parser.isSet("skipped")) {
testcase.attr("name") = "skipped test case";
failure.attr("message") = "skipped due to previous failure";
testcase<<failure;
for (int i(testsuite.children()); i<expectedtestcases; ++i)
testsuite<<testcase;
for (int i(testsuites->last().children()); i<expectedtestcases; ++i)
testsuites->last()<<testcase;
failed = true;
}
}
if (expectedtestcases==-1) {
testsuite.attr("tests") = mrw::string(testsuite.children());
testsuites->last().attr("tests") =
mrw::string(testsuites->last().children());
} else {
testsuite.attr("tests") = mrw::string(expectedtestcases);
testsuites->last().attr("tests") = mrw::string(expectedtestcases);
}
if (script.cout().size())
testsuite<<(xml::String("system-out") =
script.xmlattr(script.cout()).toStdString());
testsuites->last()<<(xml::String("system-out") =
script.xmlattr(script.cout()).toStdString());
if (script.cerr().size())
testsuite<<(xml::String("system-err") =
script.xmlattr(script.cerr()).toStdString());
testsuites<<testsuite;
testsuites->last()<<(xml::String("system-err") =
script.xmlattr(script.cerr()).toStdString());
}
if (parser.isSet("xml")) { // todo: write xml file
std::ofstream xmlfile(parser.value("xml").toStdString());
xmlfile<<testsuites<<std::endl;
xmlfile<<(*testsuites)<<std::endl;
}
return failed ? 1 : 0;
} catch (std::exception& e) {