diff --git a/web/regression/.coveragerc.in b/web/regression/.coveragerc.in new file mode 100644 index 000000000..40ced5ded --- /dev/null +++ b/web/regression/.coveragerc.in @@ -0,0 +1,25 @@ +# .coveragerc to control coverage.py +# This is default coverage configuration file require to run coverage.py +# Any paths mentioned for coverage command are indented(default:4 spaces) + +[run] +# Mention paths in 'source' to measure code coverage(Full project path) +source = + $PGADMIN4_SRC/web/pgadmin/ + +# Mention patterns in 'omit' to omit code from the coverage measurement +# Here, omit code which contains pattern '/tests/', 'feature_tests' etc. +omit = + */tests/* + */feature_tests/* + +# Mention patterns in 'include' to include code for the coverage measurement +# Here, code coverage will show report only for modules mentioned in the +# 'include' +include = + # For all modules + */web/pgadmin/* + # For 'databases' + # */web/pgadmin/browser/server_groups/servers/databases/* + # For 'tables' + # */web/pgadmin/browser/server_groups/servers/databases/schemas/tables/* diff --git a/web/regression/README b/web/regression/README index bcb8002f2..1513f6bbd 100644 --- a/web/regression/README +++ b/web/regression/README @@ -190,6 +190,42 @@ Python Tests: Example 2) Execute only reverse engineered SQL test framework for some modules run 'python runtests.py --pkg resql --modules sequences,functions' +Code Coverage: +--------------- + +- Test framework is able to calculate the code coverage. +- Coverage package(coverage) is added in $PGADMIN4_SRC/web/regression/requirements.txt file + +How to generate code coverage report for API test-suite? +------------------------------------- + +- Change to the regression test directory: + run 'cd $PGADMIN4_SRC/web/regression' + +- Before running code coverage we need configure 'regression/.coveragerc' file. + i). Create 'regression/.coveragerc' file. + ii). Copy content of 'regression/.coveragerc.in' to 'regression/.coveragerc' + iii). Modify 'regression/.coveragerc' file as per our need as 'regression/ + .coveragerc.in' has default configurations + + In 'regression/.coveragerc' file we need to mention some parameters + like 'source'(project path),'include'(files/modules to be included for + coverage),'omit'(files/modules to be omitted from coverage) + + We can also add more parameters according to our need. + For more info please read coverage.py's official document here + 'http://coverage.readthedocs.io/en/coverage-4.2/install.html' + +- Run coverage + With all modules + run 'python runtests.py --coverage --exclude feature_tests' + With specific module + run 'python runtests.py --pkg browser.server_groups.servers.tests --coverage' + +- After execution of coverage, we will see code coverage report on console. + For a nicer presentation, '/regression/covhtml' directory gets created. + Open 'index.html' file in browser and you will see detail coverage report. + Javascript Tests: - Install node.js and yarn, in the appropriate way for your platform. On macOS with Homebrew you might use: diff --git a/web/regression/python_test_utils/test_utils.py b/web/regression/python_test_utils/test_utils.py index 5d182ece2..332e17fdc 100644 --- a/web/regression/python_test_utils/test_utils.py +++ b/web/regression/python_test_utils/test_utils.py @@ -14,6 +14,7 @@ import sys import uuid import psycopg2 import sqlite3 +import shutil from functools import partial from testtools.testcase import clone_test_with_new_id @@ -23,7 +24,11 @@ from regression import test_setup from pgadmin.utils.preferences import Preferences +CURRENT_PATH = os.path.abspath(os.path.join(os.path.dirname( + os.path.realpath(__file__)), "../")) + SERVER_GROUP = test_setup.config_data['server_group'] +COVERAGE_CONFIG_FILE = os.path.join(CURRENT_PATH, ".coveragerc") file_name = os.path.realpath(__file__) @@ -1146,3 +1151,27 @@ def get_watcher_dialogue_status(self): def get_driver_version(): version = getattr(psycopg2, '__version__', None) return version + + +def is_coverage_enabled(args): + """ + This function checks for coverage args exists in command line args + :return: boolean + """ + if "coverage" in args and args["coverage"]: + return True + return False + + +def print_and_store_coverage_report(cov): + """ + This function print the coverage report on console and store it in html + files + :return: None + """ + print("\nCoverage Summary:\n", file=sys.stderr) + cov.report() + cov_dir = os.path.join(CURRENT_PATH, "covhtml") + if os.path.exists(cov_dir): + shutil.rmtree(cov_dir) + cov.html_report(directory=cov_dir) diff --git a/web/regression/requirements.txt b/web/regression/requirements.txt index ed516a1de..1eb922b19 100644 --- a/web/regression/requirements.txt +++ b/web/regression/requirements.txt @@ -27,7 +27,7 @@ testscenarios==0.5.0 testtools==2.3.0 traceback2==1.4.0 selenium==3.14.0 - +coverage==5.0.1 ############################################################### # Modules specifically required for Python3.3 or lesser version ############################################################### diff --git a/web/regression/runtests.py b/web/regression/runtests.py index f80eb5c7a..f16268c27 100644 --- a/web/regression/runtests.py +++ b/web/regression/runtests.py @@ -20,6 +20,7 @@ import sys import traceback import json import random +import coverage import unittest @@ -48,6 +49,8 @@ if sys.path[0] != root: from pgadmin import create_app import config +COVERAGE_CONFIG_FILE = os.path.join(CURRENT_PATH, ".coveragerc") + if config.SERVER_MODE is True: config.SECURITY_RECOVERABLE = True config.SECURITY_CHANGEABLE = True @@ -297,6 +300,8 @@ def add_arguments(): help='Skips execution of the test cases of particular package and ' 'sub-packages' ) + parser.add_argument('--coverage', nargs='?', const=True, type=bool, + default=False, help='Enable code coverage feature') parser.add_argument( '--default_browser', help='Executes the feature test in specific browser' @@ -394,6 +399,7 @@ if __name__ == '__main__': # Failure detected? failure = False test_result = dict() + cov = None # Set signal handler for cleanup signal_list = dir(signal) @@ -430,6 +436,12 @@ if __name__ == '__main__': node_name = "all" if args['pkg'] is not None: node_name = args['pkg'].split('.')[-1] + + # Start coverage + if test_utils.is_coverage_enabled(args): + cov = coverage.Coverage(config_file=COVERAGE_CONFIG_FILE) + cov.start() + try: for server in servers_info: print("\n=============Running the test cases for '%s'=============" @@ -557,6 +569,15 @@ if __name__ == '__main__': file=sys.stderr ) + # Stop code coverage + if test_utils.is_coverage_enabled(args): + cov.stop() + cov.save() + + # # Print coverage only if coverage args given in command line + if test_utils.is_coverage_enabled(args): + test_utils.print_and_store_coverage_report(cov) + print("Please check output in file: %s/regression.log\n" % CURRENT_PATH) # Unset environment variable