{"action":"created","comment":{"url":"https://api.github.com/repos/AtteqCom/zsl/pulls/comments/103064179","pull_request_review_id":23844432,"id":103064179,"diff_hunk":"@@ -0,0 +1,101 @@\n+\"\"\"\n+:mod:`zsl.interface.celery.worker`\n+----------------------------------\n+\n+Implementation of celery workers.\n+\n+.. moduleauthor:: Peter Morihladko\n+\"\"\"\n+from __future__ import unicode_literals\n+\n+import sys\n+\n+from celery import Celery, shared_task\n+from injector import Module, singleton, provides\n+\n+from zsl.utils.injection_helper import inject\n+from zsl.interface.task_queue import TaskQueueWorker\n+from zsl.task.job_context import Job\n+\n+\n+class CeleryTaskQueueWorkerBase(TaskQueueWorker):\n+    \"\"\"Base class for celery task queue worker.\n+\n+    It contains only the task execution part of worker.\n+    \"\"\"\n+\n+    def execute_celery_task(self, job_data):\n+        # type: (dict) -> dict\n+        \"\"\"Creates job from task and executes the job.\n+\n+        :param job_data: job's data\n+        :return: job results\n+        :rtype: dict\n+        \"\"\"\n+        job = Job(job_data)\n+\n+        return self.execute_job(job)\n+\n+\n+class CeleryTaskQueueOutsideWorker(CeleryTaskQueueWorkerBase):\n+    \"\"\"Celery worker used only for task execution.\n+\n+    This can be used when the worker is manage with some other application,\n+    like `celery worker` or `celery multi`.\n+    \"\"\"\n+\n+    def stop_worker(self):\n+        self._app.logger.error(\"Running from celery worker, kill from shell!\")\n+\n+    def run(self):\n+        self._app.logger.error(\"Running from celery worker, start from shell!\")\n+\n+\n+class CeleryTaskQueueMainWorker(CeleryTaskQueueWorkerBase):\n+    \"\"\"Worker implementation for Celery task queue.\"\"\"\n+\n+    def __init__(self,):\n+        super(CeleryTaskQueueMainWorker, self).__init__()\n+\n+        self.celery_app = Celery()\n+        self.celery_app.config_from_object(self._config['CELERY'])\n+        self.celery_worker = None\n+\n+    def stop_worker(self):\n+        self._app.logger.info(\"Stopping Celery worker on demand - quitting.\")\n+        self.celery_worker.stop()\n+\n+    def run(self, argv, *args, **kwargs):\n+        self._app.logger.info(\"Running the worker.\")\n+        self.celery_worker = self.celery_app.worker_main((sys.argv[0],) + argv)\n+\n+\n+class CeleryTaskQueueMainWorkerModule(Module):\n+    \"\"\"Injector module for celery main worker.\"\"\"\n+    @singleton\n+    @provides(CeleryTaskQueueWorkerBase)\n+    def provide_worker(self):\n+        return CeleryTaskQueueMainWorker()\n+\n+\n+class CeleryTaskQueueOutsideWorkerModule(Module):\n+    \"\"\"Injector module for celery outside worker.\"\"\"\n+    @singleton\n+    @provides(CeleryTaskQueueWorkerBase)\n+    def provide_worker(self):\n+        return CeleryTaskQueueMainWorker()\n+\n+\n+@shared_task\n+@inject(worker=CeleryTaskQueueWorkerBase)\n+def zsl_task(job_data, worker):","path":"zsl/interface/celery/worker.py","position":91,"original_position":91,"commit_id":"866e4411e8d238a5178bb292a98e5f9d99ad53a3","original_commit_id":"866e4411e8d238a5178bb292a98e5f9d99ad53a3","user":{"login":"morihladko","id":125012,"avatar_url":"https://avatars.githubusercontent.com/u/125012?v=3","gravatar_id":"","url":"https://api.github.com/users/morihladko","html_url":"https://github.com/morihladko","followers_url":"https://api.github.com/users/morihladko/followers","following_url":"https://api.github.com/users/morihladko/following{/other_user}","gists_url":"https://api.github.com/users/morihladko/gists{/gist_id}","starred_url":"https://api.github.com/users/morihladko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/morihladko/subscriptions","organizations_url":"https://api.github.com/users/morihladko/orgs","repos_url":"https://api.github.com/users/morihladko/repos","events_url":"https://api.github.com/users/morihladko/events{/privacy}","received_events_url":"https://api.github.com/users/morihladko/received_events","type":"User","site_admin":false},"body":"For now the task execution is tied with the worker instance itself, I used injection to get the current, single instance. ","created_at":"2017-02-25T01:13:51Z","updated_at":"2017-02-25T01:13:51Z","html_url":"https://github.com/AtteqCom/zsl/pull/22#discussion_r103064179","pull_request_url":"https://api.github.com/repos/AtteqCom/zsl/pulls/22","_links":{"self":{"href":"https://api.github.com/repos/AtteqCom/zsl/pulls/comments/103064179"},"html":{"href":"https://github.com/AtteqCom/zsl/pull/22#discussion_r103064179"},"pull_request":{"href":"https://api.github.com/repos/AtteqCom/zsl/pulls/22"}}},"pull_request":{"url":"https://api.github.com/repos/AtteqCom/zsl/pulls/22","id":107922904,"html_url":"https://github.com/AtteqCom/zsl/pull/22","diff_url":"https://github.com/AtteqCom/zsl/pull/22.diff","patch_url":"https://github.com/AtteqCom/zsl/pull/22.patch","issue_url":"https://api.github.com/repos/AtteqCom/zsl/issues/22","number":22,"state":"open","locked":false,"title":"Support for other task queues","user":{"login":"morihladko","id":125012,"avatar_url":"https://avatars.githubusercontent.com/u/125012?v=3","gravatar_id":"","url":"https://api.github.com/users/morihladko","html_url":"https://github.com/morihladko","followers_url":"https://api.github.com/users/morihladko/followers","following_url":"https://api.github.com/users/morihladko/following{/other_user}","gists_url":"https://api.github.com/users/morihladko/gists{/gist_id}","starred_url":"https://api.github.com/users/morihladko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/morihladko/subscriptions","organizations_url":"https://api.github.com/users/morihladko/orgs","repos_url":"https://api.github.com/users/morihladko/repos","events_url":"https://api.github.com/users/morihladko/events{/privacy}","received_events_url":"https://api.github.com/users/morihladko/received_events","type":"User","site_admin":false},"body":"I abstracted the Worker class from GearmanWorker and implemented a Celery worker onto it. There are two Celery implementation, one to run as standalone worker with help of zsl and one to expose the only the celery task function to be run with other tools (celeryd..).\r\n\r\n```bash\r\n# this will run the CeleryTaskQueueMainWorker\r\n$ python -m zsl worker celery\r\n```\r\n\r\n```bash\r\n# this will run the CeleryTaskQueueOutsideWorker\r\n$ celery worker -A app.celery\r\n```\r\n\r\n```python\r\n# app.cellery\r\nfrom celery import Celery\r\n\r\nfrom zsl.interface.importer import initialize_web_application\r\ninitialize_web_application()\r\n\r\nfrom zsl.application.service_application import service_application\r\n\r\nfrom zsl.interface.celery.worker import CeleryTaskQueueOutsideWorkerModule\r\n\r\nservice_application.add_injector_module(CeleryTaskQueueOutsideWorkerModule)\r\n\r\napp = Celery(backend='rpc', broker='redis://localhost')\r\n```\r\n\r\nWhile working on the run script for the gearman worker I refactored the run package to use pocoo's click library for the cli calls.\r\n\r\nAlso I started to define \"stable\" API in the main `zsl` package.\r\n\r\nAnd started remove initializers they only append zsl package to python path, which is not needed anymore.","created_at":"2017-02-24T22:42:02Z","updated_at":"2017-02-25T01:13:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"df54261cb62f51090bb6bd63cfb1ffe2908b187f","assignee":null,"assignees":[],"milestone":{"url":"https://api.github.com/repos/AtteqCom/zsl/milestones/1","html_url":"https://github.com/AtteqCom/zsl/milestone/1","labels_url":"https://api.github.com/repos/AtteqCom/zsl/milestones/1/labels","id":2255431,"number":1,"title":"phase 2","description":"","creator":{"login":"morihladko","id":125012,"avatar_url":"https://avatars.githubusercontent.com/u/125012?v=3","gravatar_id":"","url":"https://api.github.com/users/morihladko","html_url":"https://github.com/morihladko","followers_url":"https://api.github.com/users/morihladko/followers","following_url":"https://api.github.com/users/morihladko/following{/other_user}","gists_url":"https://api.github.com/users/morihladko/gists{/gist_id}","starred_url":"https://api.github.com/users/morihladko/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/morihladko/subscriptions","organizations_url":"https://api.github.com/users/morihladko/orgs","repos_url":"https://api.github.com/users/morihladko/repos","events_url":"https://api.github.com/users/morihladko/events{/privacy}","received_events_url":"https://api.github.com/users/morihladko/received_events","type":"User","site_admin":false},"open_issues":7,"closed_issues":0,"state":"open","created_at":"2017-01-16T17:16:47Z","updated_at":"2017-02-24T22:42:02Z","due_on":"2017-02-20T08:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/AtteqCom/zsl/pulls/22/commits","review_comments_url":"https://api.github.com/repos/AtteqCom/zsl/pulls/22/comments","review_comment_url":"https://api.github.com/repos/AtteqCom/zsl/pulls/comments{/number}","comments_url":"https://api.github.com/repos/AtteqCom/zsl/issues/22/comments","statuses_url":"https://api.github.com/repos/AtteqCom/zsl/statuses/866e4411e8d238a5178bb292a98e5f9d99ad53a3","head":{"label":"AtteqCom:7-support-for-other-task-queues","ref":"7-support-for-other-task-queues","sha":"866e4411e8d238a5178bb292a98e5f9d99ad53a3","user":{"login":"AtteqCom","id":23108265,"avatar_url":"https://avatars.githubusercontent.com/u/23108265?v=3","gravatar_id":"","url":"https://api.github.com/users/AtteqCom","html_url":"https://github.com/AtteqCom","followers_url":"https://api.github.com/users/AtteqCom/followers","following_url":"https://api.github.com/users/AtteqCom/following{/other_user}","gists_url":"https://api.github.com/users/AtteqCom/gists{/gist_id}","starred_url":"https://api.github.com/users/AtteqCom/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AtteqCom/subscriptions","organizations_url":"https://api.github.com/users/AtteqCom/orgs","repos_url":"https://api.github.com/users/AtteqCom/repos","events_url":"https://api.github.com/users/AtteqCom/events{/privacy}","received_events_url":"https://api.github.com/users/AtteqCom/received_events","type":"Organization","site_admin":false},"repo":{"id":75487836,"name":"zsl","full_name":"AtteqCom/zsl","owner":{"login":"AtteqCom","id":23108265,"avatar_url":"https://avatars.githubusercontent.com/u/23108265?v=3","gravatar_id":"","url":"https://api.github.com/users/AtteqCom","html_url":"https://github.com/AtteqCom","followers_url":"https://api.github.com/users/AtteqCom/followers","following_url":"https://api.github.com/users/AtteqCom/following{/other_user}","gists_url":"https://api.github.com/users/AtteqCom/gists{/gist_id}","starred_url":"https://api.github.com/users/AtteqCom/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AtteqCom/subscriptions","organizations_url":"https://api.github.com/users/AtteqCom/orgs","repos_url":"https://api.github.com/users/AtteqCom/repos","events_url":"https://api.github.com/users/AtteqCom/events{/privacy}","received_events_url":"https://api.github.com/users/AtteqCom/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/AtteqCom/zsl","description":"zsl application framework for web based services","fork":false,"url":"https://api.github.com/repos/AtteqCom/zsl","forks_url":"https://api.github.com/repos/AtteqCom/zsl/forks","keys_url":"https://api.github.com/repos/AtteqCom/zsl/keys{/key_id}","collaborators_url":"https://api.github.com/repos/AtteqCom/zsl/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/AtteqCom/zsl/teams","hooks_url":"https://api.github.com/repos/AtteqCom/zsl/hooks","issue_events_url":"https://api.github.com/repos/AtteqCom/zsl/issues/events{/number}","events_url":"https://api.github.com/repos/AtteqCom/zsl/events","assignees_url":"https://api.github.com/repos/AtteqCom/zsl/assignees{/user}","branches_url":"https://api.github.com/repos/AtteqCom/zsl/branches{/branch}","tags_url":"https://api.github.com/repos/AtteqCom/zsl/tags","blobs_url":"https://api.github.com/repos/AtteqCom/zsl/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/AtteqCom/zsl/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/AtteqCom/zsl/git/refs{/sha}","trees_url":"https://api.github.com/repos/AtteqCom/zsl/git/trees{/sha}","statuses_url":"https://api.github.com/repos/AtteqCom/zsl/statuses/{sha}","languages_url":"https://api.github.com/repos/AtteqCom/zsl/languages","stargazers_url":"https://api.github.com/repos/AtteqCom/zsl/stargazers","contributors_url":"https://api.github.com/repos/AtteqCom/zsl/contributors","subscribers_url":"https://api.github.com/repos/AtteqCom/zsl/subscribers","subscription_url":"https://api.github.com/repos/AtteqCom/zsl/subscription","commits_url":"https://api.github.com/repos/AtteqCom/zsl/commits{/sha}","git_commits_url":"https://api.github.com/repos/AtteqCom/zsl/git/commits{/sha}","comments_url":"https://api.github.com/repos/AtteqCom/zsl/comments{/number}","issue_comment_url":"https://api.github.com/repos/AtteqCom/zsl/issues/comments{/number}","contents_url":"https://api.github.com/repos/AtteqCom/zsl/contents/{+path}","compare_url":"https://api.github.com/repos/AtteqCom/zsl/compare/{base}...{head}","merges_url":"https://api.github.com/repos/AtteqCom/zsl/merges","archive_url":"https://api.github.com/repos/AtteqCom/zsl/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/AtteqCom/zsl/downloads","issues_url":"https://api.github.com/repos/AtteqCom/zsl/issues{/number}","pulls_url":"https://api.github.com/repos/AtteqCom/zsl/pulls{/number}","milestones_url":"https://api.github.com/repos/AtteqCom/zsl/milestones{/number}","notifications_url":"https://api.github.com/repos/AtteqCom/zsl/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/AtteqCom/zsl/labels{/name}","releases_url":"https://api.github.com/repos/AtteqCom/zsl/releases{/id}","deployments_url":"https://api.github.com/repos/AtteqCom/zsl/deployments","created_at":"2016-12-03T17:07:43Z","updated_at":"2016-12-03T17:19:53Z","pushed_at":"2017-02-25T01:08:28Z","git_url":"git://github.com/AtteqCom/zsl.git","ssh_url":"git@github.com:AtteqCom/zsl.git","clone_url":"https://github.com/AtteqCom/zsl.git","svn_url":"https://github.com/AtteqCom/zsl","homepage":null,"size":12330,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":17,"forks":0,"open_issues":17,"watchers":0,"default_branch":"master"}},"base":{"label":"AtteqCom:master","ref":"master","sha":"0d28f324a8ff076dee28a4074b77c5f0dc005e4b","user":{"login":"AtteqCom","id":23108265,"avatar_url":"https://avatars.githubusercontent.com/u/23108265?v=3","gravatar_id":"","url":"https://api.github.com/users/AtteqCom","html_url":"https://github.com/AtteqCom","followers_url":"https://api.github.com/users/AtteqCom/followers","following_url":"https://api.github.com/users/AtteqCom/following{/other_user}","gists_url":"https://api.github.com/users/AtteqCom/gists{/gist_id}","starred_url":"https://api.github.com/users/AtteqCom/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AtteqCom/subscriptions","organizations_url":"https://api.github.com/users/AtteqCom/orgs","repos_url":"https://api.github.com/users/AtteqCom/repos","events_url":"https://api.github.com/users/AtteqCom/events{/privacy}","received_events_url":"https://api.github.com/users/AtteqCom/received_events","type":"Organization","site_admin":false},"repo":{"id":75487836,"name":"zsl","full_name":"AtteqCom/zsl","owner":{"login":"AtteqCom","id":23108265,"avatar_url":"https://avatars.githubusercontent.com/u/23108265?v=3","gravatar_id":"","url":"https://api.github.com/users/AtteqCom","html_url":"https://github.com/AtteqCom","followers_url":"https://api.github.com/users/AtteqCom/followers","following_url":"https://api.github.com/users/AtteqCom/following{/other_user}","gists_url":"https://api.github.com/users/AtteqCom/gists{/gist_id}","starred_url":"https://api.github.com/users/AtteqCom/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AtteqCom/subscriptions","organizations_url":"https://api.github.com/users/AtteqCom/orgs","repos_url":"https://api.github.com/users/AtteqCom/repos","events_url":"https://api.github.com/users/AtteqCom/events{/privacy}","received_events_url":"https://api.github.com/users/AtteqCom/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/AtteqCom/zsl","description":"zsl application framework for web based services","fork":false,"url":"https://api.github.com/repos/AtteqCom/zsl","forks_url":"https://api.github.com/repos/AtteqCom/zsl/forks","keys_url":"https://api.github.com/repos/AtteqCom/zsl/keys{/key_id}","collaborators_url":"https://api.github.com/repos/AtteqCom/zsl/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/AtteqCom/zsl/teams","hooks_url":"https://api.github.com/repos/AtteqCom/zsl/hooks","issue_events_url":"https://api.github.com/repos/AtteqCom/zsl/issues/events{/number}","events_url":"https://api.github.com/repos/AtteqCom/zsl/events","assignees_url":"https://api.github.com/repos/AtteqCom/zsl/assignees{/user}","branches_url":"https://api.github.com/repos/AtteqCom/zsl/branches{/branch}","tags_url":"https://api.github.com/repos/AtteqCom/zsl/tags","blobs_url":"https://api.github.com/repos/AtteqCom/zsl/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/AtteqCom/zsl/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/AtteqCom/zsl/git/refs{/sha}","trees_url":"https://api.github.com/repos/AtteqCom/zsl/git/trees{/sha}","statuses_url":"https://api.github.com/repos/AtteqCom/zsl/statuses/{sha}","languages_url":"https://api.github.com/repos/AtteqCom/zsl/languages","stargazers_url":"https://api.github.com/repos/AtteqCom/zsl/stargazers","contributors_url":"https://api.github.com/repos/AtteqCom/zsl/contributors","subscribers_url":"https://api.github.com/repos/AtteqCom/zsl/subscribers","subscription_url":"https://api.github.com/repos/AtteqCom/zsl/subscription","commits_url":"https://api.github.com/repos/AtteqCom/zsl/commits{/sha}","git_commits_url":"https://api.github.com/repos/AtteqCom/zsl/git/commits{/sha}","comments_url":"https://api.github.com/repos/AtteqCom/zsl/comments{/number}","issue_comment_url":"https://api.github.com/repos/AtteqCom/zsl/issues/comments{/number}","contents_url":"https://api.github.com/repos/AtteqCom/zsl/contents/{+path}","compare_url":"https://api.github.com/repos/AtteqCom/zsl/compare/{base}...{head}","merges_url":"https://api.github.com/repos/AtteqCom/zsl/merges","archive_url":"https://api.github.com/repos/AtteqCom/zsl/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/AtteqCom/zsl/downloads","issues_url":"https://api.github.com/repos/AtteqCom/zsl/issues{/number}","pulls_url":"https://api.github.com/repos/AtteqCom/zsl/pulls{/number}","milestones_url":"https://api.github.com/repos/AtteqCom/zsl/milestones{/number}","notifications_url":"https://api.github.com/repos/AtteqCom/zsl/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/AtteqCom/zsl/labels{/name}","releases_url":"https://api.github.com/repos/AtteqCom/zsl/releases{/id}","deployments_url":"https://api.github.com/repos/AtteqCom/zsl/deployments","created_at":"2016-12-03T17:07:43Z","updated_at":"2016-12-03T17:19:53Z","pushed_at":"2017-02-25T01:08:28Z","git_url":"git://github.com/AtteqCom/zsl.git","ssh_url":"git@github.com:AtteqCom/zsl.git","clone_url":"https://github.com/AtteqCom/zsl.git","svn_url":"https://github.com/AtteqCom/zsl","homepage":null,"size":12330,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":17,"forks":0,"open_issues":17,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/AtteqCom/zsl/pulls/22"},"html":{"href":"https://github.com/AtteqCom/zsl/pull/22"},"issue":{"href":"https://api.github.com/repos/AtteqCom/zsl/issues/22"},"comments":{"href":"https://api.github.com/repos/AtteqCom/zsl/issues/22/comments"},"review_comments":{"href":"https://api.github.com/repos/AtteqCom/zsl/pulls/22/comments"},"review_comment":{"href":"https://api.github.com/repos/AtteqCom/zsl/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/AtteqCom/zsl/pulls/22/commits"},"statuses":{"href":"https://api.github.com/repos/AtteqCom/zsl/statuses/866e4411e8d238a5178bb292a98e5f9d99ad53a3"}}}}
{"actor":{"display_login":"morihladko"}}