Add regression tests (#41)

Now regression tests for resolved issues can be added to the project.
Regression tests are enabled by default.
They can be manually run by appending `regression-tests` to the `run-tests.py` command.
There is an `expected-failures` directory for tests that should fail.
The .gitignore file was modified as well to making sure it doesn't ignore these testfiles.

Signed-off-by: Daniel Balla dballa@inf.u-szeged.hu

.gitignore

@@ -11,6 +11,8 @@ output.js
 *.js
 *.swp
 #!test/*.js
+!test/regression-tests/*.js
+!test/regression-tests/xfail/*.js
 *.pyc
 tags
 cscope.out

.travis.yml

@@ -23,7 +23,7 @@ matrix:
         - ninja -Cout/darwin/x64/release
         - cp ./out/darwin/x64/release/escargot ./escargot
         - travis_wait 30 tools/run-tests.py --arch=x86_64 jetstream-only-simple
-        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js test262 internal octane jsc-stress
+        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js test262 internal octane jsc-stress regression-tests
         # FIXME: chakracore fails on darwin
         # ChakraCore's test runner uses `readlink -f` to determine the test root directory.
         # However, `readlink` has no `-f` option on macOS.
@@ -58,7 +58,7 @@ matrix:
         - ninja -Cout/linux/x64/release
         - cp ./out/linux/x64/release/escargot ./escargot
         - travis_wait tools/run-tests.py --arch=x86_64 jetstream-only-simple
-        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js test262 internal octane chakracore jsc-stress
+        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js test262 internal octane chakracore jsc-stress regression-tests
 
     - name: "linux.x64.debug"
       addons:
@@ -68,7 +68,7 @@ matrix:
         - cmake -H. -Bout/linux/x64/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -GNinja
         - ninja -Cout/linux/x64/debug
         - cp ./out/linux/x64/debug/escargot ./escargot
-        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js test262 internal
+        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js test262 internal regression-tests
 
     - name: "linux.x86.release"
       addons:
@@ -89,7 +89,7 @@ matrix:
         - cmake -H. -Bout/linux/x86/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -GNinja
         - ninja -Cout/linux/x86/debug
         - cp ./out/linux/x86/debug/escargot ./escargot
-        - tools/run-tests.py --arch=x86 jetstream-only-cdjs sunspider-js test262 internal
+        - tools/run-tests.py --arch=x86 jetstream-only-cdjs sunspider-js test262 internal regression-tests
 
     - name: "linux.x64.release (-DVENDORTEST=1)"
       addons:
@@ -113,7 +113,7 @@ matrix:
         - cmake -H. -Bout/linux/x86/release -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja
         - ninja -Cout/linux/x86/release
         - cp ./out/linux/x86/release/escargot ./escargot
-        - tools/run-tests.py --arch=x86 v8 spidermonkey
+        - tools/run-tests.py --arch=x86 v8 spidermonkey regression-tests
 
     - name: "SonarQube"
       addons:

test/regression-tests/assert.js

@@ -0,0 +1,20 @@
+/* Copyright 2019-present Samsung Electronics Co., Ltd. and other contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function assert(expression) {
+  if (!expression) {
+    throw new Error("Assertion failed");
+  }
+}

test/regression-tests/issue-28.js

@@ -0,0 +1,23 @@
+/* Copyright 2019-present Samsung Electronics Co., Ltd. and other contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var proto = { };
+Object.defineProperty(proto, "2", { get: function ( ) { }, configurable: true });
+Object.defineProperty(proto, "2", { writable: false });
+
+var desc = Object.getOwnPropertyDescriptor(proto, "2");
+assert(desc.value === undefined);
+assert(desc.writable === false);
+assert(desc.configurable === true);

test/regression-tests/issue-31.js

@@ -0,0 +1,16 @@
+/* Copyright 2019-present Samsung Electronics Co., Ltd. and other contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+assert(JSON.stringify(new Uint32Array()) === '{}');

test/regression-tests/issue-33.js

@@ -0,0 +1,19 @@
+/* Copyright 2019-present Samsung Electronics Co., Ltd. and other contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var arrObj = [ ] ;
+Object.preventExtensions( arrObj , "length" , { writable : false } ) ;
+Object.defineProperty( arrObj , "length" , { value : 12 } ) ;
+assert(arrObj.toString() == ",,,,,,,,,,,");

test/regression-tests/issue-5.js

@@ -0,0 +1,20 @@
+/* Copyright 2018-present Samsung Electronics Co., Ltd. and other contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Function.prototype.length = function() { };
+Function.prototype.bind(0);
+
+assert(Function.length == 1);
+assert(Function.prototype.length == 0);

test/regression-tests/xfail/issue-2.js

@@ -0,0 +1,16 @@
+/* Copyright 2018-present Samsung Electronics Co., Ltd. and other contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+new Float32Array({ length: 0x40000001 });

test/regression-tests/xfail/issue-36.js

@@ -0,0 +1,17 @@
+/* Copyright 2019-present Samsung Electronics Co., Ltd. and other contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Error.prototype.toString = Object.getOwnPropertyDescriptor.toString;
+throw ReferenceError("x");

tools/run-tests.py

@@ -202,6 +202,47 @@ def run_jsc_stress(engine, arch):
         env={'PYTHONPATH': '.'})
 
 
+def _run_regression_tests(engine, assert_js, files, is_fail):
+    fails = 0
+    for file in files:
+        proc = Popen([engine, assert_js, file], stdout=PIPE)
+        out, _ = proc.communicate()
+
+        if is_fail and proc.returncode or not is_fail and not proc.returncode:
+            print('%sOK: %s%s' % (COLOR_GREEN, file, COLOR_RESET))
+        else:
+            print('%sFAIL(%d): %s%s' % (COLOR_RED, proc.returncode, file, COLOR_RESET))
+            print(out)
+
+            fails += 1
+
+    return fails
+
+
+@runner('regression-tests', default=True)
+def run_regression_tests(engine, arch):
+    REGRESSION_DIR = join(PROJECT_SOURCE_DIR, 'test', 'regression-tests')
+    REGRESSION_XFAIL_DIR = join(REGRESSION_DIR, 'xfail')
+    REGRESSION_ASSERT_JS = join(REGRESSION_DIR, 'assert.js')
+
+    print('Running regression tests:')
+    xpass = glob(join(REGRESSION_DIR, 'issue-*.js'))
+    xpass_result = _run_regression_tests(engine, REGRESSION_ASSERT_JS, xpass, False)
+
+    print('Running regression tests expected to fail:')
+    xfail = glob(join(REGRESSION_XFAIL_DIR, 'issue-*.js'))
+    xfail_result = _run_regression_tests(engine, REGRESSION_ASSERT_JS, xfail, True)
+
+    tests_total = len(xpass) + len(xfail)
+    fail_total = xfail_result + xpass_result
+    print('TOTAL: %d' % (tests_total))
+    print('%sPASS : %d%s' % (COLOR_GREEN, tests_total - fail_total, COLOR_RESET))
+    print('%sFAIL : %d%s' % (COLOR_RED, fail_total, COLOR_RESET))
+
+    if fail_total > 0:
+        raise Exception("Regression tests failed")
+
+
 def _run_jetstream(engine, target_test):
     JETSTREAM_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'driver', 'jetstream')
     JETSTREAM_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'JetStream-1.1')