{"action":"created","comment":{"url":"https://api.github.com/repos/scality/clueso/pulls/comments/151106560","pull_request_review_id":76759174,"id":151106560,"diff_hunk":"@@ -0,0 +1,129 @@\n+package com.scality.clueso\n+\n+import java.io.{File, IOException}\n+import java.net.URI\n+\n+import com.scality.clueso.query.{MetadataQuery, MetadataQueryExecutor}\n+import com.typesafe.config.ConfigFactory\n+import com.typesafe.scalalogging.LazyLogging\n+import org.apache.hadoop.conf.{Configuration => HadoopConfig}\n+import org.apache.hadoop.fs.{FileStatus, FileSystem, Path, PathFilter}\n+import org.apache.spark.sql.functions.{col, lit}\n+import org.apache.spark.sql.{Row, SparkSession}\n+\n+\n+object SparkUtils extends LazyLogging {\n+  def loadCluesoConfig(confFilePath: String) = {\n+    val parsedConfig = ConfigFactory.parseFile(new File(confFilePath))\n+    val _config = ConfigFactory.load(parsedConfig)\n+\n+    new CluesoConfig(_config)\n+  }\n+\n+  def buildHadoopFs(config: CluesoConfig) = {\n+    FileSystem.get(new URI(PathUtils.landingURI(config)), hadoopConfig(config))\n+  }\n+\n+  def hadoopConfig(config: CluesoConfig): HadoopConfig = {\n+    val c = new HadoopConfig()\n+    c.set(\"fs.s3a.connection.ssl.enabled\", config.s3SslEnabled)\n+    c.set(\"fs.s3a.endpoint\", config.s3Endpoint)\n+    c.set(\"fs.s3a.impl\", \"org.apache.hadoop.fs.s3a.S3AFileSystem\")\n+    c.set(\"fs.s3a.access.key\", config.s3AccessKey)\n+    c.set(\"fs.s3a.secret.key\", config.s3SecretKey)\n+    c.set(\"fs.s3a.path.style.access\", config.s3PathStyleAccess)\n+    c.set(\"spark.serializer\", \"org.apache.spark.serializer.KryoSerializer\")","path":"src/main/scala/com/scality/clueso/SparkUtils.scala","position":35,"original_position":35,"commit_id":"c60580a8340ef68258775142f9ab1010014f6e20","original_commit_id":"31689a0dc4bf21c8223435f503017caf0294d424","user":{"login":"deorus","id":1385635,"avatar_url":"https://avatars0.githubusercontent.com/u/1385635?v=4","gravatar_id":"","url":"https://api.github.com/users/deorus","html_url":"https://github.com/deorus","followers_url":"https://api.github.com/users/deorus/followers","following_url":"https://api.github.com/users/deorus/following{/other_user}","gists_url":"https://api.github.com/users/deorus/gists{/gist_id}","starred_url":"https://api.github.com/users/deorus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deorus/subscriptions","organizations_url":"https://api.github.com/users/deorus/orgs","repos_url":"https://api.github.com/users/deorus/repos","events_url":"https://api.github.com/users/deorus/events{/privacy}","received_events_url":"https://api.github.com/users/deorus/received_events","type":"User","site_admin":false},"body":"Now it's defined on spark-defaults.conf in livy:/spark/conf . I cannot yet say how good it is, will do a simulation with the 7m records like before to do an apples to apples comparison","created_at":"2017-11-15T11:59:22Z","updated_at":"2017-11-15T11:59:22Z","html_url":"https://github.com/scality/clueso/pull/2#discussion_r151106560","pull_request_url":"https://api.github.com/repos/scality/clueso/pulls/2","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/scality/clueso/pulls/comments/151106560"},"html":{"href":"https://github.com/scality/clueso/pull/2#discussion_r151106560"},"pull_request":{"href":"https://api.github.com/repos/scality/clueso/pulls/2"}},"in_reply_to_id":150369826},"pull_request":{"url":"https://api.github.com/repos/scality/clueso/pulls/2","id":151809903,"html_url":"https://github.com/scality/clueso/pull/2","diff_url":"https://github.com/scality/clueso/pull/2.diff","patch_url":"https://github.com/scality/clueso/pull/2.patch","issue_url":"https://api.github.com/repos/scality/clueso/issues/2","number":2,"state":"open","locked":false,"title":"Package dev","user":{"login":"deorus","id":1385635,"avatar_url":"https://avatars0.githubusercontent.com/u/1385635?v=4","gravatar_id":"","url":"https://api.github.com/users/deorus","html_url":"https://github.com/deorus","followers_url":"https://api.github.com/users/deorus/followers","following_url":"https://api.github.com/users/deorus/following{/other_user}","gists_url":"https://api.github.com/users/deorus/gists{/gist_id}","starred_url":"https://api.github.com/users/deorus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deorus/subscriptions","organizations_url":"https://api.github.com/users/deorus/orgs","repos_url":"https://api.github.com/users/deorus/repos","events_url":"https://api.github.com/users/deorus/events{/privacy}","received_events_url":"https://api.github.com/users/deorus/received_events","type":"User","site_admin":false},"body":"","created_at":"2017-11-10T02:03:50Z","updated_at":"2017-11-15T11:59:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7e9fe8bb4048320f5f281f36f36cabddfeb9e666","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/scality/clueso/pulls/2/commits","review_comments_url":"https://api.github.com/repos/scality/clueso/pulls/2/comments","review_comment_url":"https://api.github.com/repos/scality/clueso/pulls/comments{/number}","comments_url":"https://api.github.com/repos/scality/clueso/issues/2/comments","statuses_url":"https://api.github.com/repos/scality/clueso/statuses/c60580a8340ef68258775142f9ab1010014f6e20","head":{"label":"scality:package-dev","ref":"package-dev","sha":"c60580a8340ef68258775142f9ab1010014f6e20","user":{"login":"scality","id":383812,"avatar_url":"https://avatars1.githubusercontent.com/u/383812?v=4","gravatar_id":"","url":"https://api.github.com/users/scality","html_url":"https://github.com/scality","followers_url":"https://api.github.com/users/scality/followers","following_url":"https://api.github.com/users/scality/following{/other_user}","gists_url":"https://api.github.com/users/scality/gists{/gist_id}","starred_url":"https://api.github.com/users/scality/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scality/subscriptions","organizations_url":"https://api.github.com/users/scality/orgs","repos_url":"https://api.github.com/users/scality/repos","events_url":"https://api.github.com/users/scality/events{/privacy}","received_events_url":"https://api.github.com/users/scality/received_events","type":"Organization","site_admin":false},"repo":{"id":98914853,"name":"clueso","full_name":"scality/clueso","owner":{"login":"scality","id":383812,"avatar_url":"https://avatars1.githubusercontent.com/u/383812?v=4","gravatar_id":"","url":"https://api.github.com/users/scality","html_url":"https://github.com/scality","followers_url":"https://api.github.com/users/scality/followers","following_url":"https://api.github.com/users/scality/following{/other_user}","gists_url":"https://api.github.com/users/scality/gists{/gist_id}","starred_url":"https://api.github.com/users/scality/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scality/subscriptions","organizations_url":"https://api.github.com/users/scality/orgs","repos_url":"https://api.github.com/users/scality/repos","events_url":"https://api.github.com/users/scality/events{/privacy}","received_events_url":"https://api.github.com/users/scality/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scality/clueso","description":"Search all the things!","fork":false,"url":"https://api.github.com/repos/scality/clueso","forks_url":"https://api.github.com/repos/scality/clueso/forks","keys_url":"https://api.github.com/repos/scality/clueso/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scality/clueso/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scality/clueso/teams","hooks_url":"https://api.github.com/repos/scality/clueso/hooks","issue_events_url":"https://api.github.com/repos/scality/clueso/issues/events{/number}","events_url":"https://api.github.com/repos/scality/clueso/events","assignees_url":"https://api.github.com/repos/scality/clueso/assignees{/user}","branches_url":"https://api.github.com/repos/scality/clueso/branches{/branch}","tags_url":"https://api.github.com/repos/scality/clueso/tags","blobs_url":"https://api.github.com/repos/scality/clueso/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scality/clueso/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scality/clueso/git/refs{/sha}","trees_url":"https://api.github.com/repos/scality/clueso/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scality/clueso/statuses/{sha}","languages_url":"https://api.github.com/repos/scality/clueso/languages","stargazers_url":"https://api.github.com/repos/scality/clueso/stargazers","contributors_url":"https://api.github.com/repos/scality/clueso/contributors","subscribers_url":"https://api.github.com/repos/scality/clueso/subscribers","subscription_url":"https://api.github.com/repos/scality/clueso/subscription","commits_url":"https://api.github.com/repos/scality/clueso/commits{/sha}","git_commits_url":"https://api.github.com/repos/scality/clueso/git/commits{/sha}","comments_url":"https://api.github.com/repos/scality/clueso/comments{/number}","issue_comment_url":"https://api.github.com/repos/scality/clueso/issues/comments{/number}","contents_url":"https://api.github.com/repos/scality/clueso/contents/{+path}","compare_url":"https://api.github.com/repos/scality/clueso/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scality/clueso/merges","archive_url":"https://api.github.com/repos/scality/clueso/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scality/clueso/downloads","issues_url":"https://api.github.com/repos/scality/clueso/issues{/number}","pulls_url":"https://api.github.com/repos/scality/clueso/pulls{/number}","milestones_url":"https://api.github.com/repos/scality/clueso/milestones{/number}","notifications_url":"https://api.github.com/repos/scality/clueso/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scality/clueso/labels{/name}","releases_url":"https://api.github.com/repos/scality/clueso/releases{/id}","deployments_url":"https://api.github.com/repos/scality/clueso/deployments","created_at":"2017-07-31T17:56:40Z","updated_at":"2017-07-31T18:08:30Z","pushed_at":"2017-11-15T00:08:16Z","git_url":"git://github.com/scality/clueso.git","ssh_url":"git@github.com:scality/clueso.git","clone_url":"https://github.com/scality/clueso.git","svn_url":"https://github.com/scality/clueso","homepage":null,"size":352,"stargazers_count":1,"watchers_count":1,"language":null,"has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":3,"forks":0,"open_issues":3,"watchers":1,"default_branch":"master"}},"base":{"label":"scality:master","ref":"master","sha":"047929ebf925b4751f6c12a58dafca1670855200","user":{"login":"scality","id":383812,"avatar_url":"https://avatars1.githubusercontent.com/u/383812?v=4","gravatar_id":"","url":"https://api.github.com/users/scality","html_url":"https://github.com/scality","followers_url":"https://api.github.com/users/scality/followers","following_url":"https://api.github.com/users/scality/following{/other_user}","gists_url":"https://api.github.com/users/scality/gists{/gist_id}","starred_url":"https://api.github.com/users/scality/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scality/subscriptions","organizations_url":"https://api.github.com/users/scality/orgs","repos_url":"https://api.github.com/users/scality/repos","events_url":"https://api.github.com/users/scality/events{/privacy}","received_events_url":"https://api.github.com/users/scality/received_events","type":"Organization","site_admin":false},"repo":{"id":98914853,"name":"clueso","full_name":"scality/clueso","owner":{"login":"scality","id":383812,"avatar_url":"https://avatars1.githubusercontent.com/u/383812?v=4","gravatar_id":"","url":"https://api.github.com/users/scality","html_url":"https://github.com/scality","followers_url":"https://api.github.com/users/scality/followers","following_url":"https://api.github.com/users/scality/following{/other_user}","gists_url":"https://api.github.com/users/scality/gists{/gist_id}","starred_url":"https://api.github.com/users/scality/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scality/subscriptions","organizations_url":"https://api.github.com/users/scality/orgs","repos_url":"https://api.github.com/users/scality/repos","events_url":"https://api.github.com/users/scality/events{/privacy}","received_events_url":"https://api.github.com/users/scality/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scality/clueso","description":"Search all the things!","fork":false,"url":"https://api.github.com/repos/scality/clueso","forks_url":"https://api.github.com/repos/scality/clueso/forks","keys_url":"https://api.github.com/repos/scality/clueso/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scality/clueso/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scality/clueso/teams","hooks_url":"https://api.github.com/repos/scality/clueso/hooks","issue_events_url":"https://api.github.com/repos/scality/clueso/issues/events{/number}","events_url":"https://api.github.com/repos/scality/clueso/events","assignees_url":"https://api.github.com/repos/scality/clueso/assignees{/user}","branches_url":"https://api.github.com/repos/scality/clueso/branches{/branch}","tags_url":"https://api.github.com/repos/scality/clueso/tags","blobs_url":"https://api.github.com/repos/scality/clueso/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scality/clueso/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scality/clueso/git/refs{/sha}","trees_url":"https://api.github.com/repos/scality/clueso/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scality/clueso/statuses/{sha}","languages_url":"https://api.github.com/repos/scality/clueso/languages","stargazers_url":"https://api.github.com/repos/scality/clueso/stargazers","contributors_url":"https://api.github.com/repos/scality/clueso/contributors","subscribers_url":"https://api.github.com/repos/scality/clueso/subscribers","subscription_url":"https://api.github.com/repos/scality/clueso/subscription","commits_url":"https://api.github.com/repos/scality/clueso/commits{/sha}","git_commits_url":"https://api.github.com/repos/scality/clueso/git/commits{/sha}","comments_url":"https://api.github.com/repos/scality/clueso/comments{/number}","issue_comment_url":"https://api.github.com/repos/scality/clueso/issues/comments{/number}","contents_url":"https://api.github.com/repos/scality/clueso/contents/{+path}","compare_url":"https://api.github.com/repos/scality/clueso/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scality/clueso/merges","archive_url":"https://api.github.com/repos/scality/clueso/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scality/clueso/downloads","issues_url":"https://api.github.com/repos/scality/clueso/issues{/number}","pulls_url":"https://api.github.com/repos/scality/clueso/pulls{/number}","milestones_url":"https://api.github.com/repos/scality/clueso/milestones{/number}","notifications_url":"https://api.github.com/repos/scality/clueso/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scality/clueso/labels{/name}","releases_url":"https://api.github.com/repos/scality/clueso/releases{/id}","deployments_url":"https://api.github.com/repos/scality/clueso/deployments","created_at":"2017-07-31T17:56:40Z","updated_at":"2017-07-31T18:08:30Z","pushed_at":"2017-11-15T00:08:16Z","git_url":"git://github.com/scality/clueso.git","ssh_url":"git@github.com:scality/clueso.git","clone_url":"https://github.com/scality/clueso.git","svn_url":"https://github.com/scality/clueso","homepage":null,"size":352,"stargazers_count":1,"watchers_count":1,"language":null,"has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":3,"forks":0,"open_issues":3,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scality/clueso/pulls/2"},"html":{"href":"https://github.com/scality/clueso/pull/2"},"issue":{"href":"https://api.github.com/repos/scality/clueso/issues/2"},"comments":{"href":"https://api.github.com/repos/scality/clueso/issues/2/comments"},"review_comments":{"href":"https://api.github.com/repos/scality/clueso/pulls/2/comments"},"review_comment":{"href":"https://api.github.com/repos/scality/clueso/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/scality/clueso/pulls/2/commits"},"statuses":{"href":"https://api.github.com/repos/scality/clueso/statuses/c60580a8340ef68258775142f9ab1010014f6e20"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"deorus"}}