Improve test generation process
authorAryeh Gregor <AryehGregor+gitcommit@gmail.com>
Tue, 20 Sep 2011 14:31:13 -0600
changeset 601 8d721ff655d2
parent 600 fc1acceae8d7
child 602 78c3c2c6467f
Improve test generation process

The new procedure is much simpler:

1) Run gentest.html in Gecko and WebKit.

2) When it finishes, click the submit button in the webpage itself,
providing an appropriate password.

3) Run conformancetest/fixdata to generate data.js.

4) Use git diff to check for changes in expectations.

This also fixes a couple of encoding-related errors in data.js. I've
checked them with a hex editor and the new versions are correct.

Also, this builds the 1M-ish test output string as a regular JS string
instead of in the DOM, at the advice of people like roc. This yields an
approximately order-of-magnitude improvement in test generation time,
which is now down to 2-4 minutes on my desktop from like 20. Seriously!
.gitignore
conformancetest/data.js
conformancetest/fixdata
conformancetest/gentest.html
conformancetest/submit.php
--- a/.gitignore	Tue Sep 20 12:49:36 2011 -0600
+++ b/.gitignore	Tue Sep 20 14:31:13 2011 -0600
@@ -1,2 +1,6 @@
 anolis
 data
+conformancetest/gecko-data
+conformancetest/webkit-data
+conformancetest/patch
+conformancetest/diff
--- a/conformancetest/data.js	Tue Sep 20 12:49:36 2011 -0600
+++ b/conformancetest/data.js	Tue Sep 20 14:31:13 2011 -0600
@@ -15248,12 +15248,12 @@
 	"<p>fooabc</p><p>def[]bar</p>",
 	{"stylewithcss":[false,false,"",false,true,""],"inserttext":[false,false,"",false,false,""]}],
 ["foo[]bar",
-	[["stylewithcss","false"],["inserttext","�"]],
-	"foo�[]bar",
+	[["stylewithcss","false"],["inserttext","í €"]],
+	"fooí €[]bar",
 	{"stylewithcss":[false,false,"",false,false,""],"inserttext":[false,false,"",false,false,""]}],
 ["foo[]bar",
-	[["stylewithcss","true"],["inserttext","�"]],
-	"foo�[]bar",
+	[["stylewithcss","true"],["inserttext","í €"]],
+	"fooí €[]bar",
 	{"stylewithcss":[false,false,"",false,true,""],"inserttext":[false,false,"",false,false,""]}],
 ["foo[]bar",
 	[["stylewithcss","false"],["inserttext","\u0007"]],
@@ -15599,11 +15599,11 @@
 	[["stylewithcss","true"],["inserttext","a"]],
 	"foo a[]",
 	{"stylewithcss":[false,false,"",false,true,""],"inserttext":[false,false,"",false,false,""]}],
-["foo []",
+["foo []",
 	[["stylewithcss","false"],["inserttext","a"]],
 	"foo a[]",
 	{"stylewithcss":[false,true,"",false,false,""],"inserttext":[false,false,"",false,false,""]}],
-["foo []",
+["foo []",
 	[["stylewithcss","true"],["inserttext","a"]],
 	"foo a[]",
 	{"stylewithcss":[false,false,"",false,true,""],"inserttext":[false,false,"",false,false,""]}],
--- a/conformancetest/fixdata	Tue Sep 20 12:49:36 2011 -0600
+++ b/conformancetest/fixdata	Tue Sep 20 14:31:13 2011 -0600
@@ -2,4 +2,13 @@
 # Currently I cross-check data generation in Gecko and WebKit, then populate
 # using the WebKit copy because it has fewer bugs to fix.  This fixes the bugs
 # for me.  Must be run from the root of the spec repo.
-sed -i 's!"fo<i>\[o</i><span style=\\"font-style:oblique\\">b]ar</span>baz"!"fo<i>[o</i><span style=\\"font-style:oblique\\"><i>b]</i>ar</span>baz"!;s!"fo<span style=\\"font-style:italic\\">\[o</span><span style=\\"font-style:oblique\\">b]ar</span>baz"!"fo<span style=\\"font-style:italic\\">[o</span><span style=\\"font-style:oblique\\"><span style=\\"font-style:italic\\">b]</span>ar</span>baz"!' conformancetest/data.js
+set -e
+cd conformancetest
+fromdos webkit-data
+fromdos gecko-data
+diff -u webkit-data gecko-data | tail -n +3 > tmp-diff
+diff -u tmp-diff diff
+rm tmp-diff
+cp webkit-data data.js
+patch data.js patch
+echo >> data.js
--- a/conformancetest/gentest.html	Tue Sep 20 12:49:36 2011 -0600
+++ b/conformancetest/gentest.html	Tue Sep 20 14:31:13 2011 -0600
@@ -11,7 +11,14 @@
 
 <div id=errors></div>
 
-<pre id=output></pre>
+<form action=submit.php method=post>
+<p><input type=submit disabled>
+<p><label>Label: <input name=label></label>
+<p><label><input type=radio name=ua value=gecko> Gecko</label>
+   <label><input type=radio name=ua value=webkit> WebKit</label>
+<p><label>Password: <input type=password name=password></label>
+<p><textarea name=data id=output readonly style=width:100%;height:20em></textarea>
+</form>
 
 <div id=test-container></div>
 
@@ -21,12 +28,19 @@
 <script>
 "use strict";
 
+if (/WebKit/.test(navigator.userAgent)) {
+	document.querySelector("[type=radio][value=webkit]").checked = true;
+} else if (/Gecko\//.test(navigator.userAgent)) {
+	document.querySelector("[type=radio][value=gecko]").checked = true;
+}
+
+var testOutput = '';
+
 function generateTests() {
 	var startTime = Date.now();
 	document.getElementById("test-container").innerHTML = "<div contenteditable></div><p>test";
 
-	var outputEl = document.getElementById("output");
-	outputEl.textContent = "var browserTests = [\n";
+	testOutput = "var browserTests = [\n";
 	for (var command in tests) {
 		tests[command].forEach(function(test) {
 			if (badTests[command].indexOf(test) != -1) {
@@ -51,15 +65,18 @@
 
 	// Finish up our JSON.  First remove the last comma, since JSON doesn't
 	// like that, then add a closing bracket.
-	outputEl.firstChild.deleteData(outputEl.textContent.lastIndexOf(","), 1);
-	outputEl.firstChild.appendData("]");
+	testOutput = testOutput.substr(0, testOutput.lastIndexOf(","))
+		+ testOutput.substr(1 + testOutput.lastIndexOf(","))
+		+ "]";
+	document.getElementById("output").value = testOutput;
+	document.getElementById("output").select();
 
 	document.getElementById("test-container").parentNode
 		.removeChild(document.getElementById("test-container"));
 	var elapsed = Math.round(Date.now() - startTime)/1000;
 	document.getElementById("timing").textContent =
 		"Time elapsed: " + Math.floor(elapsed/60) + ":" + (elapsed % 60).toFixed(3) + " min.";
-	getSelection().selectAllChildren(document.getElementById("output"));
+	document.querySelector("[type=submit]").disabled = false;
 }
 
 // Helper function for generateTest()
@@ -116,8 +133,6 @@
  * The converted test value is then added to the pre with id=output.
  */
 function generateTest(test) {
-	var output = document.getElementById("output").firstChild;
-
 	var testDiv = document.querySelector("div[contenteditable]");
 	var points = setupDiv(testDiv, test[0]);
 
@@ -223,9 +238,9 @@
 
 	addBrackets(range);
 
-	output.appendData("[" + JSON.stringify(test[0]) + ",\n"
+	testOutput += "[" + JSON.stringify(test[0]) + ",\n"
 		+ "\t" + JSON.stringify(test.slice(1)) + ",\n"
 		+ "\t" + JSON.stringify(testDiv.innerHTML) + ",\n"
-		+ "\t" + JSON.stringify(expectedQueryResults) + "],\n");
+		+ "\t" + JSON.stringify(expectedQueryResults) + "],\n";
 }
 </script>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/conformancetest/submit.php	Tue Sep 20 14:31:13 2011 -0600
@@ -0,0 +1,17 @@
+<?php
+# I hate PHP, but it's what I know, so this is fastest . . .
+if (sha1($_POST['password']) !== 'd962ad564032fa99ca43e8f0f6875c8efb9e2905') {
+	# I love how hash functions let me leave the source code open with no
+	# database or secret files, without disclosing the password.
+	die('Incorrect password');
+}
+
+if ($_POST['ua'] !== 'webkit' && $_POST['ua'] !== 'gecko') {
+	die('No UA provided: must be either "webkit" or "gecko"');
+}
+
+if (file_put_contents($_POST['ua'] . '-data', $_POST['data']) === false) {
+	die('Write failed!');
+}
+
+echo 'Successfully wrote ' . $_POST['ua'] . '-data for ' . $_POST['label'];