{"action":"created","comment":{"url":"https://api.github.com/repos/LLNL/spack/pulls/comments/97472617","pull_request_review_id":18080609,"id":97472617,"diff_hunk":"@@ -0,0 +1,221 @@\n+#!/usr/bin/env python\n+\n+import os\n+import requests\n+import glob\n+import argparse\n+import llnl.util.tty as tty\n+from spack.spec import Spec\n+import spack.cmd.install as install    \n+import spack.cmd.uninstall as uninstall\n+import spack.util.spack_yaml as yaml\n+import spack.compilers \n+import spack\n+\n+description = \"Compiles a list of tests from a yaml file. Runs Spec and concretize then produces cdash format.\"\n+\n+#not sure if I will use this later cdash or junit, spack.io or whereever or other flags\n+def setup_parser(subparser):\n+    subparser.add_argument(\n+        '-c', '--complete', action='store_true', dest='complete',\n+        help='using this option switches from simple cdash output to compelet: simple is only build, complete is configure, build and test xml output.')\n+    subparser.add_argument(\n+        'yamlFile', nargs=argparse.REMAINDER,\n+        help=\"yaml file that contains a list of tests, example yaml file can be found in /lib/spack/docs/tutorial/examples/test.yaml\")\n+\n+'''\n+fucntion Name: reduceCompilerList\n+parameters:\n+    masterList : list\n+returns:\n+    list\n+Get the list of compilers from spack found on the system.\n+Compares the spack list to test file.\n+If a compiler is found on both lists its returned.\n+'''\n+def reduceCompilerList(masterList):\n+    systemCompilers = []\n+    newMaster = []\n+    newList = []\n+    for item in masterList:\n+        newMaster.append(str(item))\n+    \n+    for compilers in spack.compilers.all_compilers(scope=None):\n+        systemCompilers.append(str(compilers))\n+\n+    for compiler in systemCompilers:\n+        if compiler in newMaster:\n+            newList.append(compiler)\n+    return newList\n+\n+\n+'''\n+fucntion Name: removeTests\n+parameters:\n+    testList  : list\n+    exclusion : str\n+returns:\n+    list\n+\n+Receieves a list of all tests and an excusion.\n+Depending on the type of exclusion it will remove tests matching.\n+Returns the list with the tests matching the excusion removed.\n+\n+exclusions can be in this form:\n+pkg%compiler\n+pkg@version\n+compiler@version\n+pkg@version%compiler@version\n+pkg@version%compiler\n+pkg%compiler@version\n+pkg\n+compiler\n+'''\n+def removeTests(testList, exclusion):\n+    pkg = \"\"\n+    compiler=\"\"\n+    teststoRemove = []\n+    if \"%\" in exclusion:\n+        pkg = exclusion.split('%')[0]\n+        compiler=exclusion.split('%')[1]\n+        if '@' in pkg and '@' in compiler:#pkg@version%compiler@version\n+            if exclusion in testList:\n+                teststoRemove.append(exclusion)\n+        elif '@' in pkg or '@' in compiler:#pkg@version%compiler or pkg%compiler@version\n+            for test in testList:\n+                if pkg in test and compiler in test:\n+                    teststoRemove.append(test)\n+        else:#pkg%compiler\n+            for test in testList:\n+                if pkg in test and compiler in test:\n+                    teststoRemove.append(test)\n+    elif '@' in exclusion: #pkg@version or compiler@version\n+        for test in testList:\n+            if exclusion in test:\n+                teststoRemove.append(test)\n+    else:#pkg or compiler\n+        remove=0\n+        for test in testList:\n+            if exclusion in str(test):\n+                teststoRemove.append(test)\n+    for test in teststoRemove: #remove tests from main test list\n+        testList.remove(test)\n+    return testList\n+\n+def testsuite(parser, args):\n+    if not args.yamlFile:\n+        tty.die(\"spack testsuite requires a yaml file as argument.\")\n+    if args.complete:\n+        args.log_format='cdash-complete' #use cdash-complete if you want configure, build and test output.\n+        cdash = '--log-format=cdash-complete'\n+    else:\n+        args.log_format='cdash-simple' #use cdash-complete if you want configure, build and test output.\n+        cdash = '--log-format=cdash-simple'\n+    print args.complete\n+    enabledTests = []\n+    compilers = []\n+    packages = []\n+    tests = []\n+    testPackages = []\n+    versions = []\n+    packageVersion = []\n+    compilerVersion = []\n+    path = \"\"\n+    dashboards = []\n+    #designed to use a single file and modify the enabled tests, thus requiring a single file modification.\n+    for files in args.yamlFile: #read yaml files which contains description of tests\n+        with open(files, 'r') as stream:\n+            try:\n+                yamlFile= yaml.load(stream)\n+            except yaml.YAMLError as exc:\n+                print(exc)\n+\n+            enabledTests= yamlFile['enable']\n+            packages= yamlFile['packages'] # list of packages to test. Current libelf, libdwarf, bzip2\n+            compilers = yamlFile['compilers']\n+            \n+            try:\n+                path=yamlFile['path']\n+            except KeyError as e:\n+                tty.msg(\"Path not found, will use \" + str(spack.prefix)+\"/var/spack/cdash/\")\n+\n+            try:\n+                dashboards = yamlFile['dashboard']\n+            except KeyError as e:\n+                if path is \"\":\n+                    tty.msg(\"dashboard url was not found. xml files will be stored at \" + str(spack.prefix)+\"/var/spack/cdash/\")\n+                else:\n+                    tty.msg(\"dashboard url was not found. xml files will be stored at path provided.\")\n+    #creating a list of packages \n+    for package in packages:\n+        for pkg in package:\n+            if pkg in enabledTests:\n+                versions = package[pkg][0]['versions']\n+                for version in versions:\n+                    # producing packages at available versions. Sample file contains only checksum'd\n+                    packageVersion.append(str(pkg)+\"@\"+str(version))\n+\n+    #creating a list of compilers\n+    for compiler in compilers:\n+        for comp in compiler:\n+            versions = compiler[comp][0]['versions']\n+            for version in versions:\n+                # producing compilers at available versions. Sample file contains only checksum'd\n+                compilerVersion.append(str(comp)+\"@\"+str(version))\n+    #reducing compiler list to whats actually available on the system\n+    compilerVersion = reduceCompilerList(compilerVersion)\n+    #producing a list of tests with a combination of packages and compilers\n+    for pkg in packageVersion:\n+        for comp in compilerVersion:\n+            tests.append(str(pkg)+\"%\"+str(comp))\n+\n+    #loading test excusions\n+    exclusions = yamlFile['exclusions']\n+    if len(exclusions) != 0:\n+        #remove test that match the exclusion\n+        for exclusion in exclusions:\n+            tests=removeTests(tests, exclusion)\n+\n+\n+    concreteTests = []\n+    \n+    #setting up tests for contretizing\n+    for test in tests:\n+        spec = Spec(test)\n+        #uninstall all packages before installing. This will reduce the number of skipped package installs.\n+        while (len(spack.store.db.query(spec)) > 0):\n+            spec.concretize()\n+            tty.msg(\"uninstalling \" + str(spec))\n+            pkg = spack.repo.get(spec)\n+            pkg.do_uninstall()\n+        #concretize, failing can occur if the package uses the wrong compiler which would produce a failure for cdash\n+        try:\n+            spec.concretize()\n+            concreteTests.append(spec.to_yaml())\n+            parser = argparse.ArgumentParser()\n+            install.setup_parser(parser)\n+            args = parser.parse_args([cdash]) #use cdash-complete if you want configure, build and test output.\n+            args.package = test\n+            install.install(parser, args)\n+        except Exception as ex:","path":"lib/spack/spack/cmd/testsuite.py","position":200,"original_position":200,"commit_id":"6ed56dd87b1bf2080851adcc41930018a6164cb9","original_commit_id":"6ed56dd87b1bf2080851adcc41930018a6164cb9","user":{"login":"tgamblin","id":299842,"avatar_url":"https://avatars.githubusercontent.com/u/299842?v=3","gravatar_id":"","url":"https://api.github.com/users/tgamblin","html_url":"https://github.com/tgamblin","followers_url":"https://api.github.com/users/tgamblin/followers","following_url":"https://api.github.com/users/tgamblin/following{/other_user}","gists_url":"https://api.github.com/users/tgamblin/gists{/gist_id}","starred_url":"https://api.github.com/users/tgamblin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tgamblin/subscriptions","organizations_url":"https://api.github.com/users/tgamblin/orgs","repos_url":"https://api.github.com/users/tgamblin/repos","events_url":"https://api.github.com/users/tgamblin/events{/privacy}","received_events_url":"https://api.github.com/users/tgamblin/received_events","type":"User","site_admin":false},"body":"Whatever error occurs should probably go in the test log and get displayed as part of CDash output.","created_at":"2017-01-24T04:28:13Z","updated_at":"2017-01-24T04:38:40Z","html_url":"https://github.com/LLNL/spack/pull/2917#discussion_r97472617","pull_request_url":"https://api.github.com/repos/LLNL/spack/pulls/2917","_links":{"self":{"href":"https://api.github.com/repos/LLNL/spack/pulls/comments/97472617"},"html":{"href":"https://github.com/LLNL/spack/pull/2917#discussion_r97472617"},"pull_request":{"href":"https://api.github.com/repos/LLNL/spack/pulls/2917"}}},"pull_request":{"url":"https://api.github.com/repos/LLNL/spack/pulls/2917","id":102831456,"html_url":"https://github.com/LLNL/spack/pull/2917","diff_url":"https://github.com/LLNL/spack/pull/2917.diff","patch_url":"https://github.com/LLNL/spack/pull/2917.patch","issue_url":"https://api.github.com/repos/LLNL/spack/issues/2917","number":2917,"state":"open","locked":false,"title":"Add spack testsuite command for posting combinatorial tests to CDash","user":{"login":"kielfriedt","id":552747,"avatar_url":"https://avatars.githubusercontent.com/u/552747?v=3","gravatar_id":"","url":"https://api.github.com/users/kielfriedt","html_url":"https://github.com/kielfriedt","followers_url":"https://api.github.com/users/kielfriedt/followers","following_url":"https://api.github.com/users/kielfriedt/following{/other_user}","gists_url":"https://api.github.com/users/kielfriedt/gists{/gist_id}","starred_url":"https://api.github.com/users/kielfriedt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kielfriedt/subscriptions","organizations_url":"https://api.github.com/users/kielfriedt/orgs","repos_url":"https://api.github.com/users/kielfriedt/repos","events_url":"https://api.github.com/users/kielfriedt/events{/privacy}","received_events_url":"https://api.github.com/users/kielfriedt/received_events","type":"User","site_admin":false},"body":"Added testsuite which reads in a yaml file. Example yaml file is located in lib/spack/docs/tutorial/examples/. Changed dashboard to simple and complete versions. Current requirements are for build only, but in the future a complete configure, build and test would be needed. Testsuite is documented and tested. ","created_at":"2017-01-24T00:39:44Z","updated_at":"2017-01-24T04:38:40Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/LLNL/spack/pulls/2917/commits","review_comments_url":"https://api.github.com/repos/LLNL/spack/pulls/2917/comments","review_comment_url":"https://api.github.com/repos/LLNL/spack/pulls/comments{/number}","comments_url":"https://api.github.com/repos/LLNL/spack/issues/2917/comments","statuses_url":"https://api.github.com/repos/LLNL/spack/statuses/6ed56dd87b1bf2080851adcc41930018a6164cb9","head":{"label":"LLNL:wip","ref":"wip","sha":"6ed56dd87b1bf2080851adcc41930018a6164cb9","user":{"login":"LLNL","id":5921419,"avatar_url":"https://avatars.githubusercontent.com/u/5921419?v=3","gravatar_id":"","url":"https://api.github.com/users/LLNL","html_url":"https://github.com/LLNL","followers_url":"https://api.github.com/users/LLNL/followers","following_url":"https://api.github.com/users/LLNL/following{/other_user}","gists_url":"https://api.github.com/users/LLNL/gists{/gist_id}","starred_url":"https://api.github.com/users/LLNL/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LLNL/subscriptions","organizations_url":"https://api.github.com/users/LLNL/orgs","repos_url":"https://api.github.com/users/LLNL/repos","events_url":"https://api.github.com/users/LLNL/events{/privacy}","received_events_url":"https://api.github.com/users/LLNL/received_events","type":"Organization","site_admin":false},"repo":{"id":15730865,"name":"spack","full_name":"LLNL/spack","owner":{"login":"LLNL","id":5921419,"avatar_url":"https://avatars.githubusercontent.com/u/5921419?v=3","gravatar_id":"","url":"https://api.github.com/users/LLNL","html_url":"https://github.com/LLNL","followers_url":"https://api.github.com/users/LLNL/followers","following_url":"https://api.github.com/users/LLNL/following{/other_user}","gists_url":"https://api.github.com/users/LLNL/gists{/gist_id}","starred_url":"https://api.github.com/users/LLNL/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LLNL/subscriptions","organizations_url":"https://api.github.com/users/LLNL/orgs","repos_url":"https://api.github.com/users/LLNL/repos","events_url":"https://api.github.com/users/LLNL/events{/privacy}","received_events_url":"https://api.github.com/users/LLNL/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/LLNL/spack","description":"A flexible package manager designed to support multiple versions, configurations, platforms, and compilers.","fork":false,"url":"https://api.github.com/repos/LLNL/spack","forks_url":"https://api.github.com/repos/LLNL/spack/forks","keys_url":"https://api.github.com/repos/LLNL/spack/keys{/key_id}","collaborators_url":"https://api.github.com/repos/LLNL/spack/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/LLNL/spack/teams","hooks_url":"https://api.github.com/repos/LLNL/spack/hooks","issue_events_url":"https://api.github.com/repos/LLNL/spack/issues/events{/number}","events_url":"https://api.github.com/repos/LLNL/spack/events","assignees_url":"https://api.github.com/repos/LLNL/spack/assignees{/user}","branches_url":"https://api.github.com/repos/LLNL/spack/branches{/branch}","tags_url":"https://api.github.com/repos/LLNL/spack/tags","blobs_url":"https://api.github.com/repos/LLNL/spack/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/LLNL/spack/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/LLNL/spack/git/refs{/sha}","trees_url":"https://api.github.com/repos/LLNL/spack/git/trees{/sha}","statuses_url":"https://api.github.com/repos/LLNL/spack/statuses/{sha}","languages_url":"https://api.github.com/repos/LLNL/spack/languages","stargazers_url":"https://api.github.com/repos/LLNL/spack/stargazers","contributors_url":"https://api.github.com/repos/LLNL/spack/contributors","subscribers_url":"https://api.github.com/repos/LLNL/spack/subscribers","subscription_url":"https://api.github.com/repos/LLNL/spack/subscription","commits_url":"https://api.github.com/repos/LLNL/spack/commits{/sha}","git_commits_url":"https://api.github.com/repos/LLNL/spack/git/commits{/sha}","comments_url":"https://api.github.com/repos/LLNL/spack/comments{/number}","issue_comment_url":"https://api.github.com/repos/LLNL/spack/issues/comments{/number}","contents_url":"https://api.github.com/repos/LLNL/spack/contents/{+path}","compare_url":"https://api.github.com/repos/LLNL/spack/compare/{base}...{head}","merges_url":"https://api.github.com/repos/LLNL/spack/merges","archive_url":"https://api.github.com/repos/LLNL/spack/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/LLNL/spack/downloads","issues_url":"https://api.github.com/repos/LLNL/spack/issues{/number}","pulls_url":"https://api.github.com/repos/LLNL/spack/pulls{/number}","milestones_url":"https://api.github.com/repos/LLNL/spack/milestones{/number}","notifications_url":"https://api.github.com/repos/LLNL/spack/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/LLNL/spack/labels{/name}","releases_url":"https://api.github.com/repos/LLNL/spack/releases{/id}","deployments_url":"https://api.github.com/repos/LLNL/spack/deployments","created_at":"2014-01-08T09:22:12Z","updated_at":"2017-01-23T14:59:56Z","pushed_at":"2017-01-24T01:48:25Z","git_url":"git://github.com/LLNL/spack.git","ssh_url":"git@github.com:LLNL/spack.git","clone_url":"https://github.com/LLNL/spack.git","svn_url":"https://github.com/LLNL/spack","homepage":"http://spack.io","size":19795,"stargazers_count":202,"watchers_count":202,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":179,"mirror_url":null,"open_issues_count":488,"forks":179,"open_issues":488,"watchers":202,"default_branch":"develop"}},"base":{"label":"LLNL:develop","ref":"develop","sha":"db7a786d1cb61ee87d84c52781740440d8632096","user":{"login":"LLNL","id":5921419,"avatar_url":"https://avatars.githubusercontent.com/u/5921419?v=3","gravatar_id":"","url":"https://api.github.com/users/LLNL","html_url":"https://github.com/LLNL","followers_url":"https://api.github.com/users/LLNL/followers","following_url":"https://api.github.com/users/LLNL/following{/other_user}","gists_url":"https://api.github.com/users/LLNL/gists{/gist_id}","starred_url":"https://api.github.com/users/LLNL/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LLNL/subscriptions","organizations_url":"https://api.github.com/users/LLNL/orgs","repos_url":"https://api.github.com/users/LLNL/repos","events_url":"https://api.github.com/users/LLNL/events{/privacy}","received_events_url":"https://api.github.com/users/LLNL/received_events","type":"Organization","site_admin":false},"repo":{"id":15730865,"name":"spack","full_name":"LLNL/spack","owner":{"login":"LLNL","id":5921419,"avatar_url":"https://avatars.githubusercontent.com/u/5921419?v=3","gravatar_id":"","url":"https://api.github.com/users/LLNL","html_url":"https://github.com/LLNL","followers_url":"https://api.github.com/users/LLNL/followers","following_url":"https://api.github.com/users/LLNL/following{/other_user}","gists_url":"https://api.github.com/users/LLNL/gists{/gist_id}","starred_url":"https://api.github.com/users/LLNL/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LLNL/subscriptions","organizations_url":"https://api.github.com/users/LLNL/orgs","repos_url":"https://api.github.com/users/LLNL/repos","events_url":"https://api.github.com/users/LLNL/events{/privacy}","received_events_url":"https://api.github.com/users/LLNL/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/LLNL/spack","description":"A flexible package manager designed to support multiple versions, configurations, platforms, and compilers.","fork":false,"url":"https://api.github.com/repos/LLNL/spack","forks_url":"https://api.github.com/repos/LLNL/spack/forks","keys_url":"https://api.github.com/repos/LLNL/spack/keys{/key_id}","collaborators_url":"https://api.github.com/repos/LLNL/spack/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/LLNL/spack/teams","hooks_url":"https://api.github.com/repos/LLNL/spack/hooks","issue_events_url":"https://api.github.com/repos/LLNL/spack/issues/events{/number}","events_url":"https://api.github.com/repos/LLNL/spack/events","assignees_url":"https://api.github.com/repos/LLNL/spack/assignees{/user}","branches_url":"https://api.github.com/repos/LLNL/spack/branches{/branch}","tags_url":"https://api.github.com/repos/LLNL/spack/tags","blobs_url":"https://api.github.com/repos/LLNL/spack/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/LLNL/spack/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/LLNL/spack/git/refs{/sha}","trees_url":"https://api.github.com/repos/LLNL/spack/git/trees{/sha}","statuses_url":"https://api.github.com/repos/LLNL/spack/statuses/{sha}","languages_url":"https://api.github.com/repos/LLNL/spack/languages","stargazers_url":"https://api.github.com/repos/LLNL/spack/stargazers","contributors_url":"https://api.github.com/repos/LLNL/spack/contributors","subscribers_url":"https://api.github.com/repos/LLNL/spack/subscribers","subscription_url":"https://api.github.com/repos/LLNL/spack/subscription","commits_url":"https://api.github.com/repos/LLNL/spack/commits{/sha}","git_commits_url":"https://api.github.com/repos/LLNL/spack/git/commits{/sha}","comments_url":"https://api.github.com/repos/LLNL/spack/comments{/number}","issue_comment_url":"https://api.github.com/repos/LLNL/spack/issues/comments{/number}","contents_url":"https://api.github.com/repos/LLNL/spack/contents/{+path}","compare_url":"https://api.github.com/repos/LLNL/spack/compare/{base}...{head}","merges_url":"https://api.github.com/repos/LLNL/spack/merges","archive_url":"https://api.github.com/repos/LLNL/spack/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/LLNL/spack/downloads","issues_url":"https://api.github.com/repos/LLNL/spack/issues{/number}","pulls_url":"https://api.github.com/repos/LLNL/spack/pulls{/number}","milestones_url":"https://api.github.com/repos/LLNL/spack/milestones{/number}","notifications_url":"https://api.github.com/repos/LLNL/spack/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/LLNL/spack/labels{/name}","releases_url":"https://api.github.com/repos/LLNL/spack/releases{/id}","deployments_url":"https://api.github.com/repos/LLNL/spack/deployments","created_at":"2014-01-08T09:22:12Z","updated_at":"2017-01-23T14:59:56Z","pushed_at":"2017-01-24T01:48:25Z","git_url":"git://github.com/LLNL/spack.git","ssh_url":"git@github.com:LLNL/spack.git","clone_url":"https://github.com/LLNL/spack.git","svn_url":"https://github.com/LLNL/spack","homepage":"http://spack.io","size":19795,"stargazers_count":202,"watchers_count":202,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":179,"mirror_url":null,"open_issues_count":488,"forks":179,"open_issues":488,"watchers":202,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/LLNL/spack/pulls/2917"},"html":{"href":"https://github.com/LLNL/spack/pull/2917"},"issue":{"href":"https://api.github.com/repos/LLNL/spack/issues/2917"},"comments":{"href":"https://api.github.com/repos/LLNL/spack/issues/2917/comments"},"review_comments":{"href":"https://api.github.com/repos/LLNL/spack/pulls/2917/comments"},"review_comment":{"href":"https://api.github.com/repos/LLNL/spack/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/LLNL/spack/pulls/2917/commits"},"statuses":{"href":"https://api.github.com/repos/LLNL/spack/statuses/6ed56dd87b1bf2080851adcc41930018a6164cb9"}}}}
{"actor":{"display_login":"tgamblin"}}