{"action":"created","comment":{"url":"https://api.github.com/repos/DataDog/datadog-agent/pulls/comments/153015526","pull_request_review_id":78954893,"id":153015526,"diff_hunk":"@@ -111,6 +117,115 @@ func LogMessage(message *C.char, logLevel C.int) *C.PyObject {\n \treturn C._none()\n }\n \n+// GetSubprocessOutput runs the subprocess and returns the output\n+//export GetSubprocessOutput\n+func GetSubprocessOutput(argv **C.char, argc, raise int) *C.PyObject {\n+\n+\t// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices\n+\tlength := int(argc)\n+\tsubprocessArgs := make([]string, length-1)\n+\tcmdSlice := (*[1 << 30]*C.char)(unsafe.Pointer(argv))[:length:length]\n+\tsubprocessCmd := C.GoString(cmdSlice[0])\n+\tfor i := 1; i < length; i++ {\n+\t\tsubprocessArgs[i-1] = C.GoString(cmdSlice[i])\n+\t}\n+\tcmd := exec.Command(subprocessCmd, subprocessArgs...)\n+\n+\truntime.LockOSThread()\n+\tdefer runtime.UnlockOSThread()\n+\tglock := C.PyGILState_Ensure()\n+\tdefer C.PyGILState_Release(glock)\n+\n+\tstdout, err := cmd.StdoutPipe()\n+\tif err != nil {\n+\t\tcErr := C.CString(fmt.Sprintf(\"internal error creating stdout pipe: %v\", err))\n+\t\tC.PyErr_SetString(C.PyExc_Exception, cErr)\n+\t\tC.free(unsafe.Pointer(cErr))\n+\t\treturn C._none()\n+\t}\n+\n+\tvar wg sync.WaitGroup\n+\tvar output []byte\n+\twg.Add(1)\n+\tgo func() {\n+\t\tdefer wg.Done()\n+\t\toutput, _ = ioutil.ReadAll(stdout)\n+\t}()\n+\n+\tstderr, err := cmd.StderrPipe()\n+\tif err != nil {\n+\t\tcErr := C.CString(fmt.Sprintf(\"internal error creating stderr pipe: %v\", err))\n+\t\tC.PyErr_SetString(C.PyExc_Exception, cErr)\n+\t\tC.free(unsafe.Pointer(cErr))\n+\t\treturn C._none()\n+\t}\n+\n+\tvar outputErr []byte\n+\twg.Add(1)\n+\tgo func() {\n+\t\tdefer wg.Done()\n+\t\toutputErr, _ = ioutil.ReadAll(stderr)\n+\t}()\n+\n+\tcmd.Start()\n+\n+\tretCode := 0\n+\terr = cmd.Wait()\n+\tif exiterr, ok := err.(*exec.ExitError); ok {\n+\t\tif status, ok := exiterr.Sys().(syscall.WaitStatus); ok {\n+\t\t\tretCode = status.ExitStatus()\n+\t\t}\n+\t}\n+\twg.Wait()\n+\n+\tif raise > 0 {\n+\t\t// raise on error\n+\t\tif len(output) == 0 {\n+\t\t\tcModuleName := C.CString(\"util\")\n+\t\t\tutilModule := C.PyImport_ImportModule(cModuleName)\n+\t\t\tC.free(unsafe.Pointer(cModuleName))\n+\t\t\tif utilModule == nil {\n+\t\t\t\tcErr := C.CString(\"unable to import subprocess empty output exception\")\n+\t\t\t\tC.PyErr_SetString(C.PyExc_Exception, cErr)\n+\t\t\t\tC.free(unsafe.Pointer(cErr))\n+\t\t\t\treturn C._none()\n+\t\t\t}\n+\t\t\tdefer C.Py_DecRef(utilModule)\n+\n+\t\t\tcExcName := C.CString(\"SubprocessOutputEmptyError\")\n+\t\t\texcClass := C.PyObject_GetAttrString(utilModule, cExcName)\n+\t\t\tC.free(unsafe.Pointer(cExcName))\n+\t\t\tif excClass == nil {\n+\t\t\t\tcErr := C.CString(\"unable to import subprocess empty output exception\")\n+\t\t\t\tC.PyErr_SetString(C.PyExc_Exception, cErr)","path":"pkg/collector/py/datadog_agent.go","position":97,"original_position":97,"commit_id":"e7e1eede2be35f72e8c35093bd5d7f41f67dd546","original_commit_id":"e7e1eede2be35f72e8c35093bd5d7f41f67dd546","user":{"login":"olivielpeau","id":4912500,"avatar_url":"https://avatars2.githubusercontent.com/u/4912500?v=4","gravatar_id":"","url":"https://api.github.com/users/olivielpeau","html_url":"https://github.com/olivielpeau","followers_url":"https://api.github.com/users/olivielpeau/followers","following_url":"https://api.github.com/users/olivielpeau/following{/other_user}","gists_url":"https://api.github.com/users/olivielpeau/gists{/gist_id}","starred_url":"https://api.github.com/users/olivielpeau/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/olivielpeau/subscriptions","organizations_url":"https://api.github.com/users/olivielpeau/orgs","repos_url":"https://api.github.com/users/olivielpeau/repos","events_url":"https://api.github.com/users/olivielpeau/events{/privacy}","received_events_url":"https://api.github.com/users/olivielpeau/received_events","type":"User","site_admin":false},"body":"not sure we should overwrite the original exception","created_at":"2017-11-24T18:24:57Z","updated_at":"2017-11-24T18:31:08Z","html_url":"https://github.com/DataDog/datadog-agent/pull/843#discussion_r153015526","pull_request_url":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/DataDog/datadog-agent/pulls/comments/153015526"},"html":{"href":"https://github.com/DataDog/datadog-agent/pull/843#discussion_r153015526"},"pull_request":{"href":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843"}}},"pull_request":{"url":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843","id":154350466,"html_url":"https://github.com/DataDog/datadog-agent/pull/843","diff_url":"https://github.com/DataDog/datadog-agent/pull/843.diff","patch_url":"https://github.com/DataDog/datadog-agent/pull/843.patch","issue_url":"https://api.github.com/repos/DataDog/datadog-agent/issues/843","number":843,"state":"closed","locked":false,"title":"[subprocess][go] run the subprocesses from go-land","user":{"login":"truthbk","id":448216,"avatar_url":"https://avatars0.githubusercontent.com/u/448216?v=4","gravatar_id":"","url":"https://api.github.com/users/truthbk","html_url":"https://github.com/truthbk","followers_url":"https://api.github.com/users/truthbk/followers","following_url":"https://api.github.com/users/truthbk/following{/other_user}","gists_url":"https://api.github.com/users/truthbk/gists{/gist_id}","starred_url":"https://api.github.com/users/truthbk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/truthbk/subscriptions","organizations_url":"https://api.github.com/users/truthbk/orgs","repos_url":"https://api.github.com/users/truthbk/repos","events_url":"https://api.github.com/users/truthbk/events{/privacy}","received_events_url":"https://api.github.com/users/truthbk/received_events","type":"User","site_admin":false},"body":"### What does this PR do?\r\n\r\nRuns the subprocess from go-land as opposed to forking from within the python interpreter. This should hopefully address memory leak issues we have experienced, as well as resolve the zombie fork issue (triggered by an obscure race-condition) we have experienced.\r\n\r\n### Motivation\r\n\r\nThe `subprocess32` approach in #830 has helped with the memory leak, but wasn't a truly satisfactory fix as it didn't quite fix the issue on windows. The alternative, setting a global lock on windows would likely be less desirable than a pure-go cross-platform fix. Hopefully this addresses that.\r\n\r\n### Additional Notes\r\n\r\nI've been unable to get the tests to fly on gitlab, they seem fine locally. Let's see what happens on CircleCI. Let's be thorough with the GIL, reference count during the review 😅 ","created_at":"2017-11-23T11:14:33Z","updated_at":"2017-11-24T18:31:08Z","closed_at":"2017-11-24T16:46:49Z","merged_at":"2017-11-24T16:46:49Z","merge_commit_sha":"fbcb0abb314115d81c0bb9e538d77373316cb16f","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843/commits","review_comments_url":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843/comments","review_comment_url":"https://api.github.com/repos/DataDog/datadog-agent/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DataDog/datadog-agent/issues/843/comments","statuses_url":"https://api.github.com/repos/DataDog/datadog-agent/statuses/e7e1eede2be35f72e8c35093bd5d7f41f67dd546","head":{"label":"DataDog:jaime/subprocessgo","ref":"jaime/subprocessgo","sha":"e7e1eede2be35f72e8c35093bd5d7f41f67dd546","user":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":49970739,"name":"datadog-agent","full_name":"DataDog/datadog-agent","owner":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/datadog-agent","description":"Datadog Agent ","fork":false,"url":"https://api.github.com/repos/DataDog/datadog-agent","forks_url":"https://api.github.com/repos/DataDog/datadog-agent/forks","keys_url":"https://api.github.com/repos/DataDog/datadog-agent/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/datadog-agent/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/datadog-agent/teams","hooks_url":"https://api.github.com/repos/DataDog/datadog-agent/hooks","issue_events_url":"https://api.github.com/repos/DataDog/datadog-agent/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/datadog-agent/events","assignees_url":"https://api.github.com/repos/DataDog/datadog-agent/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/datadog-agent/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/datadog-agent/tags","blobs_url":"https://api.github.com/repos/DataDog/datadog-agent/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/datadog-agent/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/datadog-agent/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/datadog-agent/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/datadog-agent/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/datadog-agent/languages","stargazers_url":"https://api.github.com/repos/DataDog/datadog-agent/stargazers","contributors_url":"https://api.github.com/repos/DataDog/datadog-agent/contributors","subscribers_url":"https://api.github.com/repos/DataDog/datadog-agent/subscribers","subscription_url":"https://api.github.com/repos/DataDog/datadog-agent/subscription","commits_url":"https://api.github.com/repos/DataDog/datadog-agent/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/datadog-agent/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/datadog-agent/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/datadog-agent/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/datadog-agent/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/datadog-agent/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/datadog-agent/merges","archive_url":"https://api.github.com/repos/DataDog/datadog-agent/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/datadog-agent/downloads","issues_url":"https://api.github.com/repos/DataDog/datadog-agent/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/datadog-agent/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/datadog-agent/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/datadog-agent/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/datadog-agent/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/datadog-agent/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/datadog-agent/deployments","created_at":"2016-01-19T17:40:41Z","updated_at":"2017-11-24T03:47:00Z","pushed_at":"2017-11-24T17:57:40Z","git_url":"git://github.com/DataDog/datadog-agent.git","ssh_url":"git@github.com:DataDog/datadog-agent.git","clone_url":"https://github.com/DataDog/datadog-agent.git","svn_url":"https://github.com/DataDog/datadog-agent","homepage":"https://docs.datadoghq.com/","size":15081,"stargazers_count":34,"watchers_count":34,"language":"Go","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":24,"forks":5,"open_issues":24,"watchers":34,"default_branch":"master"}},"base":{"label":"DataDog:master","ref":"master","sha":"f87ad76112cbb92293b20edc7661a803f07c805f","user":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"repo":{"id":49970739,"name":"datadog-agent","full_name":"DataDog/datadog-agent","owner":{"login":"DataDog","id":365230,"avatar_url":"https://avatars1.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/DataDog","html_url":"https://github.com/DataDog","followers_url":"https://api.github.com/users/DataDog/followers","following_url":"https://api.github.com/users/DataDog/following{/other_user}","gists_url":"https://api.github.com/users/DataDog/gists{/gist_id}","starred_url":"https://api.github.com/users/DataDog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataDog/subscriptions","organizations_url":"https://api.github.com/users/DataDog/orgs","repos_url":"https://api.github.com/users/DataDog/repos","events_url":"https://api.github.com/users/DataDog/events{/privacy}","received_events_url":"https://api.github.com/users/DataDog/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataDog/datadog-agent","description":"Datadog Agent ","fork":false,"url":"https://api.github.com/repos/DataDog/datadog-agent","forks_url":"https://api.github.com/repos/DataDog/datadog-agent/forks","keys_url":"https://api.github.com/repos/DataDog/datadog-agent/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataDog/datadog-agent/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataDog/datadog-agent/teams","hooks_url":"https://api.github.com/repos/DataDog/datadog-agent/hooks","issue_events_url":"https://api.github.com/repos/DataDog/datadog-agent/issues/events{/number}","events_url":"https://api.github.com/repos/DataDog/datadog-agent/events","assignees_url":"https://api.github.com/repos/DataDog/datadog-agent/assignees{/user}","branches_url":"https://api.github.com/repos/DataDog/datadog-agent/branches{/branch}","tags_url":"https://api.github.com/repos/DataDog/datadog-agent/tags","blobs_url":"https://api.github.com/repos/DataDog/datadog-agent/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataDog/datadog-agent/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataDog/datadog-agent/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataDog/datadog-agent/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataDog/datadog-agent/statuses/{sha}","languages_url":"https://api.github.com/repos/DataDog/datadog-agent/languages","stargazers_url":"https://api.github.com/repos/DataDog/datadog-agent/stargazers","contributors_url":"https://api.github.com/repos/DataDog/datadog-agent/contributors","subscribers_url":"https://api.github.com/repos/DataDog/datadog-agent/subscribers","subscription_url":"https://api.github.com/repos/DataDog/datadog-agent/subscription","commits_url":"https://api.github.com/repos/DataDog/datadog-agent/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataDog/datadog-agent/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataDog/datadog-agent/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataDog/datadog-agent/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataDog/datadog-agent/contents/{+path}","compare_url":"https://api.github.com/repos/DataDog/datadog-agent/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataDog/datadog-agent/merges","archive_url":"https://api.github.com/repos/DataDog/datadog-agent/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataDog/datadog-agent/downloads","issues_url":"https://api.github.com/repos/DataDog/datadog-agent/issues{/number}","pulls_url":"https://api.github.com/repos/DataDog/datadog-agent/pulls{/number}","milestones_url":"https://api.github.com/repos/DataDog/datadog-agent/milestones{/number}","notifications_url":"https://api.github.com/repos/DataDog/datadog-agent/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataDog/datadog-agent/labels{/name}","releases_url":"https://api.github.com/repos/DataDog/datadog-agent/releases{/id}","deployments_url":"https://api.github.com/repos/DataDog/datadog-agent/deployments","created_at":"2016-01-19T17:40:41Z","updated_at":"2017-11-24T03:47:00Z","pushed_at":"2017-11-24T17:57:40Z","git_url":"git://github.com/DataDog/datadog-agent.git","ssh_url":"git@github.com:DataDog/datadog-agent.git","clone_url":"https://github.com/DataDog/datadog-agent.git","svn_url":"https://github.com/DataDog/datadog-agent","homepage":"https://docs.datadoghq.com/","size":15081,"stargazers_count":34,"watchers_count":34,"language":"Go","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":24,"forks":5,"open_issues":24,"watchers":34,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843"},"html":{"href":"https://github.com/DataDog/datadog-agent/pull/843"},"issue":{"href":"https://api.github.com/repos/DataDog/datadog-agent/issues/843"},"comments":{"href":"https://api.github.com/repos/DataDog/datadog-agent/issues/843/comments"},"review_comments":{"href":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843/comments"},"review_comment":{"href":"https://api.github.com/repos/DataDog/datadog-agent/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DataDog/datadog-agent/pulls/843/commits"},"statuses":{"href":"https://api.github.com/repos/DataDog/datadog-agent/statuses/e7e1eede2be35f72e8c35093bd5d7f41f67dd546"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"olivielpeau"}}