Updated Branches: refs/heads/marvin c1d34369a -> c78292e9c
CLOUDSTACk-5674: Added few misc changes to make it work. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/7cc9423b Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/7cc9423b Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/7cc9423b Branch: refs/heads/marvin Commit: 7cc9423b8d1546b4a4e73a4be3a3247f41fc032a Parents: c1d3436 Author: Santhosh Edukulla <santhosh.eduku...@citrix.com> Authored: Wed Feb 12 17:33:13 2014 +0530 Committer: Girish Shilamkar <gir...@clogeny.com> Committed: Wed Feb 12 17:33:13 2014 +0530 ---------------------------------------------------------------------- tools/marvin/marvin/asyncJobMgr.py | 2 +- tools/marvin/marvin/cloudstackConnection.py | 2 +- tools/marvin/marvin/cloudstackTestClient.py | 14 ++-- tools/marvin/marvin/dbConnection.py | 2 +- tools/marvin/marvin/deployDataCenter.py | 2 +- tools/marvin/marvin/lib/utils.py | 2 +- tools/marvin/marvin/marvinInit.py | 69 ++++++++++++++------ tools/marvin/marvin/marvinLog.py | 20 +++--- tools/marvin/marvin/marvinPlugin.py | 81 ++++++++++++------------ tools/marvin/marvin/sshClient.py | 2 +- 10 files changed, 110 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/asyncJobMgr.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/asyncJobMgr.py b/tools/marvin/marvin/asyncJobMgr.py index dab85ca..ee3ae5a 100644 --- a/tools/marvin/marvin/asyncJobMgr.py +++ b/tools/marvin/marvin/asyncJobMgr.py @@ -16,7 +16,7 @@ # under the License. import threading -import cloudstackException +from marvin import cloudstackException import time import Queue import copy http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/cloudstackConnection.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py index b8d9a38..ee1ff00 100644 --- a/tools/marvin/marvin/cloudstackConnection.py +++ b/tools/marvin/marvin/cloudstackConnection.py @@ -38,7 +38,7 @@ from requests import ( Timeout, RequestException ) -from cloudstackException import GetDetailExceptionInfo +from marvin.cloudstackException import GetDetailExceptionInfo class CSConnection(object): http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/cloudstackTestClient.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py index e8c6ab2..1769e39 100644 --- a/tools/marvin/marvin/cloudstackTestClient.py +++ b/tools/marvin/marvin/cloudstackTestClient.py @@ -26,26 +26,24 @@ from codes import (FAILED, PASS, ADMIN, DOMAIN_ADMIN, USER, SUCCESS, XEN_SERVER) from configGenerator import ConfigManager from marvin.lib import utils -from cloudstackException import GetDetailExceptionInfo +from marvin.cloudstackException import GetDetailExceptionInfo from marvin.lib.utils import (random_gen, validateList) from marvin.cloudstackAPI.cloudstackAPIClient import CloudStackAPIClient -''' -@Desc : CloudStackTestClient is encapsulated entity for creating and +class CSTestClient(object): + ''' + @Desc : CloudStackTestClient is encapsulated entity for creating and getting various clients viz., apiclient, user api client, dbconnection, test Data parsed information etc -@Input : mgmtDetails : Management Server Details + @Input : mgmtDetails : Management Server Details dbSvrDetails: Database Server details of Management \ Server. Retrieved from configuration file. asyncTimeout : Timeout for Async queries defaultWorkerThreads : Number of worker threads logger : provides logging facilities for this library zone : The zone on which test suites using this test client will run -''' - - -class CSTestClient(object): + ''' def __init__(self, mgmt_details, dbsvr_details, async_timeout=3600, http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/dbConnection.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/dbConnection.py b/tools/marvin/marvin/dbConnection.py index 422fcfa..ba7bac9 100644 --- a/tools/marvin/marvin/dbConnection.py +++ b/tools/marvin/marvin/dbConnection.py @@ -20,7 +20,7 @@ import contextlib from mysql import connector from mysql.connector import errors from contextlib import closing -import cloudstackException +from marvin import cloudstackException import sys import os http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/deployDataCenter.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index 862a1bf..c8feaaf 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -686,7 +686,7 @@ if __name__ == "__main__": log_check = False if log_obj is not None: log_check = True - ret = log_obj.createLogs("DataCenter", + ret = log_obj.createLogs("DeployDataCenter", cfg.logger) if ret != FAILED: log_folder_path = log_obj.getLogFolderPath() http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/lib/utils.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/lib/utils.py b/tools/marvin/marvin/lib/utils.py index 957807d..440ff77 100644 --- a/tools/marvin/marvin/lib/utils.py +++ b/tools/marvin/marvin/lib/utils.py @@ -30,7 +30,7 @@ import urlparse import datetime from platform import system from marvin.cloudstackAPI import cloudstackAPIClient, listHosts -from cloudstackException import GetDetailExceptionInfo +from marvin.cloudstackException import GetDetailExceptionInfo from marvin.sshClient import SshClient from marvin.codes import ( SUCCESS, http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/marvinInit.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/marvinInit.py b/tools/marvin/marvin/marvinInit.py index 5218d23..131d1ef 100644 --- a/tools/marvin/marvin/marvinInit.py +++ b/tools/marvin/marvin/marvinInit.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. ''' -@Desc: Initializes the marvin and does required prerequisites +Initializes the marvin and does required prerequisites for starting it. 1. Parses the configuration file passed to marvin and creates a parsed config @@ -25,8 +25,7 @@ for starting it. ''' -from marvin import configGenerator -from marvin import cloudstackException +from marvin.configGenerator import getSetupConfig from marvin.marvinLog import MarvinLog from marvin.deployDataCenter import DeployDataCenters from marvin.cloudstackTestClient import CSTestClient @@ -49,7 +48,8 @@ from marvin.codegenerator import CodeGenerator class MarvinInit: - def __init__(self, config_file, load_api_flag=None, + def __init__(self, config_file, + load_api_flag=None, deploy_dc_flag=None, test_module_name=None, zone=None): @@ -67,14 +67,14 @@ class MarvinInit: def __parseConfig(self): ''' + @Name: __parseConfig @Desc : Parses the configuration file passed and assigns the parsed configuration + @Output : SUCCESS or FAILED ''' try: - if self.__configFile is None: - return FAILED - self.__parsedConfig = configGenerator.\ - getSetupConfig(self.__configFile) + self.__parsedConfig = getSetupConfig(self.__configFile) + return SUCCESS except Exception, e: print "\nException Occurred Under __parseConfig : " \ @@ -94,6 +94,11 @@ class MarvinInit: return self.__tcRunLogger def getDebugFile(self): + ''' + @Name : getDebugFile + @Desc : Returns the Result file to be used for writing + test outputs + ''' if self.__logFolderPath is not None: self.__tcResultFile = open(self.__logFolderPath + "/results.txt", "w") @@ -108,6 +113,7 @@ class MarvinInit: 2. Creates a timestamped log folder and provides all logs to be dumped there 3. Creates the DataCenter based upon configuration provided + @Output : SUCCESS or FAILED ''' try: if ((self.__parseConfig() != FAILED) and @@ -125,9 +131,9 @@ class MarvinInit: return FAILED def __initLogging(self): - try: - ''' - @Desc : 1. Initializes the logging for marvin and so provides + ''' + @Name : __initLogging + @Desc : 1. Initializes the logging for marvin and so provides various log features for automation run. 2. Initializes all logs to be available under given Folder Path,where all test run logs @@ -135,18 +141,21 @@ class MarvinInit: 3. All logging like exception log,results, run info etc for a given test run are available under a given timestamped folder - ''' + @Output : SUCCESS or FAILED + ''' + try: log_obj = MarvinLog("CSLog") if log_obj is None: return FAILED else: ret = log_obj.\ - getLogs(self.__testModuleName, + createLogs(self.__testModuleName, self.__parsedConfig.logger) if ret != FAILED: self.__logFolderPath = log_obj.getLogFolderPath() self.__tcRunLogger = log_obj.getLogger() - return SUCCESS + return SUCCESS + return FAILED except Exception, e: print "\n Exception Occurred Under __initLogging " \ ":%s" % GetDetailExceptionInfo(e) @@ -157,6 +166,7 @@ class MarvinInit: @Name : __createTestClient @Desc : Creates the TestClient during init based upon the parameters provided + @Output: Returns SUCCESS or FAILED ''' try: mgt_details = self.__parsedConfig.mgtSvr[0] @@ -176,7 +186,12 @@ class MarvinInit: return FAILED def __loadNewApiFromXml(self): + ''' + @Desc: Kept for future usage + Will enhance later. + ''' try: + return SUCCESS if self.__loadApiFlag: apiLoadCfg = self.__parsedConfig.apiLoadCfg api_dst_dir = apiLoadCfg.ParsedApiDestFolder + "/cloudstackAPI" @@ -189,7 +204,7 @@ class MarvinInit: print "Failed to create folder %s, " \ "due to %s" % (api_dst_dir, GetDetailExceptionInfo(e)) - exit(1) + return FAILED mgt_details = self.__parsedConfig.mgtSvr[0] cg = CodeGenerator(api_dst_dir) if os.path.exists(api_spec_file): @@ -206,6 +221,11 @@ class MarvinInit: return FAILED def __setTestDataPath(self): + ''' + @Name : __setTestDataPath + @Desc: Sets the TestData Path for tests to run + @Output: Returns SUCCESS or FAILED + ''' try: if ((self.__parsedConfig.TestData is not None) and (self.__parsedConfig.TestData.Path is not None)): @@ -217,14 +237,23 @@ class MarvinInit: return FAILED def __deployDC(self): + ''' + @Name : __deployDC + @Desc : Deploy the DataCenter and returns accordingly. + @Output : SUCCESS or FAILED + ''' try: - ''' - Deploy the DataCenter and retrieves test client. - ''' - deploy_obj = DeployDataCenters(self.__testClient, + ret = SUCCESS + if self.__deployFlag: + deploy_obj = DeployDataCenters(self.__testClient, self.__parsedConfig, self.__tcRunLogger) - return deploy_obj.deploy() if self.__deployFlag else FAILED + ret = deploy_obj.deploy() + if ret == SUCCESS: + print "Deploy DC Successful" + else: + print "Deploy DC Failed" + return ret except Exception, e: print "\n Exception Occurred Under __deployDC : %s" % \ GetDetailExceptionInfo(e) http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/marvinLog.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/marvinLog.py b/tools/marvin/marvin/marvinLog.py index 4161eca..18d705b 100644 --- a/tools/marvin/marvin/marvinLog.py +++ b/tools/marvin/marvin/marvinLog.py @@ -68,6 +68,7 @@ class MarvinLog: def __setLogHandler(self, log_file_path, log_format=None, log_level=logging.DEBUG): ''' + @Name : __setLogHandler @Desc: Adds the given Log handler to the current logger @Input: log_file_path: Log File Path as where to store the logs log_format : Format of log messages to be dumped @@ -100,7 +101,8 @@ class MarvinLog: @Input: logfolder_to_remove: Path of Log to remove ''' try: - os.rmdir(logfolder_to_remove) + if os.path.isdir(logfolder_to_remove): + os.rmdir(logfolder_to_remove) except Exception, e: print "\n Exception Occurred Under __cleanPreviousLogs :%s" % \ GetDetailExceptionInfo(e) @@ -131,21 +133,15 @@ class MarvinLog: @Output : SUCCESS\FAILED ''' try: - if log_cfg is None: - print "\nInvalid Log Folder Configuration." \ - "Please Check Config File" - return FAILED - if test_module_name is None: - temp_path = time.strftime("%b_%d_%Y_%H_%M_%S", + temp_ts = time.strftime("%b_%d_%Y_%H_%M_%S", time.localtime()) + if test_module_name is None: + temp_path = temp_ts else: - temp_path = str(test_module_name.split(".py")[0]) + temp_path = str(test_module_name) + "__" + str(temp_ts) - if (('LogFolderPath' in log_cfg.__dict__.keys()) and + if ((log_cfg is not None) and ('LogFolderPath' in log_cfg.__dict__.keys()) and (log_cfg.__dict__.get('LogFolderPath') is not None)): - self.__cleanPreviousLogs(log_cfg. - __dict__. - get('LogFolderPath') + "/MarvinLogs") temp_dir = \ log_cfg.__dict__.get('LogFolderPath') + "/MarvinLogs" else: http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/marvinPlugin.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/marvinPlugin.py b/tools/marvin/marvin/marvinPlugin.py index f36bca8..9cb3ee3 100644 --- a/tools/marvin/marvin/marvinPlugin.py +++ b/tools/marvin/marvin/marvinPlugin.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - import marvin import sys import logging @@ -25,7 +24,7 @@ from nose.plugins.base import Plugin from marvin.codes import (SUCCESS, FAILED, EXCEPTION) -from marvin.cloudstackException import GetDetailExceptionInfo +from marvin.cloudstackException import GetDetailExceptionInfo import time import os @@ -56,7 +55,7 @@ class MarvinPlugin(Plugin): ''' Signifies the flag whether to deploy the New DC or Not ''' - self.__deployDcFlag + self.__deployDcFlag = None self.conf = None self.debugStream = sys.stdout self.testRunner = None @@ -79,11 +78,26 @@ class MarvinPlugin(Plugin): else: self.enabled = True - self.__configFile = options.config_file + self.__configFile = options.configFile self.__loadNewApiFlag = options.loadNewApiFlag self.__deployDcFlag = options.deployDc self.__zoneForTests = options.zone self.conf = conf + test_mod_name = None + if self.startMarvin(test_mod_name) == FAILED: + print "\nExiting Marvin" + exit(1) + + def getModName(self): + if len(self.conf.testNames) == 0: + dir_path = getattr(self.conf.options,'where') + if dir_path is not None: + temp = os.path.split(dir_path[0].strip()) + return temp + else: + first_entry = self.conf.testNames[0] + temp = os.path.split(first_entry) + return os.path.splitext(temp)[0] def options(self, parser, env): """ @@ -92,7 +106,7 @@ class MarvinPlugin(Plugin): parser.add_option("--marvin-config", action="store", default=env.get('MARVIN_CONFIG', './datacenter.cfg'), - dest="config_file", + dest="configFile", help="Marvin's configuration file is required." "The config file containing the datacenter and " "other management server " @@ -102,7 +116,7 @@ class MarvinPlugin(Plugin): dest="deployDc", help="Deploys the DC with Given Configuration." "Requires only when DC needs to be deployed") - parser.add_option("--zone", action="zone_tests", + parser.add_option("--zone", action="store_true", default=None, dest="zone", help="Runs all tests against this specified zone") @@ -111,13 +125,6 @@ class MarvinPlugin(Plugin): dest="loadNewApiFlag", help="Loads the New Apis with Given Api Xml File." "Creates the new Api's from commands.xml File") - ''' - Check if the configuration file is not valid,print and exit - ''' - (options, args) = parser.parse_args() - if options.config_file is None: - parser.print_usage() - sys.exit(1) Plugin.options(self, parser, env) def wantClass(self, cls): @@ -127,17 +134,10 @@ class MarvinPlugin(Plugin): return True return None - def prepareTest(self, test): - ''' - @Desc : Initializes the marvin with required settings - ''' - test_module_name = test.__str__() - if self.startMarvin(test_module_name) == FAILED: - print "Starting Marvin FAILED. Please Check Config and " \ - "Arguments Supplied" - + def __checkImport(self, filename): ''' + @Name : __checkImport @Desc : Verifies to Import the test Module before running and check whether if it is importable. This will check for test modules which has some issues to be @@ -146,6 +146,7 @@ class MarvinPlugin(Plugin): ''' try: __import__(filename) + print "\n*******************", filename return True except ImportError, e: self.tcRunLogger.exception("Module : %s Import " @@ -157,14 +158,13 @@ class MarvinPlugin(Plugin): ''' @Desc : Only python files will be used as test modules ''' - if filename is None or filename == '': - return False - parts = filename.split(os.path.sep) - base, ext = os.path.splitext(parts[-1]) - if ext != '.py': - return False - else: - return self.__checkImport(filename) + print "\n*******************", filename + if os.path.isfile(filename): + if os.path.splitext(filename) != ".py": + return False + else: + return self.__checkImport(filename) + return False def loadTestsFromTestCase(self, cls): if cls.__name__ != 'cloudstackTestCase': @@ -192,34 +192,33 @@ class MarvinPlugin(Plugin): ''' Adds Exception throwing test cases and information to log. ''' - err_msg = GetDetailExceptionInfo(err) self.tcRunLogger.fatal("%s: %s: %s" % - (EXCEPTION, self.testName, err_msg)) + (EXCEPTION, self.testName, GetDetailExceptionInfo(err))) self.testResult = EXCEPTION def handleFailure(self, test, err): ''' Adds Failing test cases and information to log. ''' - err_msg = GetDetailExceptionInfo(err) self.tcRunLogger.fatal("%s: %s: %s" % - (FAILED, self.testName, err_msg)) + (FAILED, self.testName, GetDetailExceptionInfo(err))) self.testResult = FAILED def startMarvin(self, test_module_name): ''' - Initializes the Marvin - creates the test Client - creates the runlogger for logging - Parses the config and creates a parsedconfig - Creates a debugstream for tc debug log + @Name : startMarvin + @Desc : Initializes the Marvin + creates the test Client + creates the runlogger for logging + Parses the config and creates a parsedconfig + Creates a debugstream for tc debug log ''' try: obj_marvininit = MarvinInit(self.__configFile, self.__loadNewApiFlag, self.__deployDcFlag, test_module_name, - self.__zoneForoTests) + self.__zoneForTests) if obj_marvininit.init() == SUCCESS: self.testClient = obj_marvininit.getTestClient() self.tcRunLogger = obj_marvininit.getLogger() @@ -230,8 +229,10 @@ class MarvinPlugin(Plugin): descriptions=True, verbosity=2, config=self.conf) + print "\nMarvin Initialization Successful" return SUCCESS else: + print "\nMarvin Initialization Failed" return FAILED except Exception, e: print "Exception Occurred under startMarvin: %s" % \ http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7cc9423b/tools/marvin/marvin/sshClient.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/sshClient.py b/tools/marvin/marvin/sshClient.py index 9b5bca2..c24477c 100644 --- a/tools/marvin/marvin/sshClient.py +++ b/tools/marvin/marvin/sshClient.py @@ -25,7 +25,7 @@ from paramiko import (BadHostKeyException, SFTPClient) import socket import time -from cloudstackException import ( +from marvin.cloudstackException import ( internalError, GetDetailExceptionInfo )
