From e1640293113074633f0e329610745ace5f94e3f5 Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Wed, 10 Apr 2019 15:19:52 -0700 Subject: [PATCH 1/4] Suggested improvements --- Pipfile | 18 ++++ Pipfile.lock | 216 ++++++++++++++++++++++++++++++++++++++++++ loggather/settings.py | 8 +- loggather/tasks.py | 10 +- requirements.in | 12 --- requirements.txt | 37 -------- 6 files changed, 239 insertions(+), 62 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock delete mode 100644 requirements.in delete mode 100644 requirements.txt diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..53506c3 --- /dev/null +++ b/Pipfile @@ -0,0 +1,18 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +celery = "*" +gunicorn = "*" +django = "*" +django-heroku = "*" +django-crispy-forms = "*" +requests = "*" +django-open-humans = "*" + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..db0aa87 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,216 @@ +{ + "_meta": { + "hash": { + "sha256": "71500f4f15429165fdb4947060e134501a3fe41dc109ca923b63beeb0d5d7148" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "amqp": { + "hashes": [ + "sha256:043beb485774ca69718a35602089e524f87168268f0d1ae115f28b88d27f92d7", + "sha256:35a3b5006ca00b21aaeec8ceea07130f07b902dd61bfe42815039835f962f5f1" + ], + "version": "==2.4.2" + }, + "arrow": { + "hashes": [ + "sha256:3397e5448952e18e1295bf047014659effa5ae8da6a5371d37ff0ddc46fa6872", + "sha256:6f54d9f016c0b7811fac9fb8c2c7fa7421d80c54dbdd75ffb12913c55db60b8a" + ], + "version": "==0.13.1" + }, + "billiard": { + "hashes": [ + "sha256:756bf323f250db8bf88462cd042c992ba60d8f5e07fc5636c24ba7d6f4261d84" + ], + "version": "==3.6.0.0" + }, + "celery": { + "hashes": [ + "sha256:4c4532aa683f170f40bd76f928b70bc06ff171a959e06e71bf35f2f9d6031ef9", + "sha256:528e56767ae7e43a16cfef24ee1062491f5754368d38fcfffa861cdb9ef219be" + ], + "index": "pypi", + "version": "==4.3.0" + }, + "certifi": { + "hashes": [ + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + ], + "version": "==2019.3.9" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "dj-database-url": { + "hashes": [ + "sha256:4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163", + "sha256:851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9" + ], + "version": "==0.5.0" + }, + "django": { + "hashes": [ + "sha256:7c3543e4fb070d14e10926189a7fcf42ba919263b7473dceaefce34d54e8a119", + "sha256:a2814bffd1f007805b19194eb0b9a331933b82bd5da1c3ba3d7b7ba16e06dc4b" + ], + "index": "pypi", + "version": "==2.2" + }, + "django-crispy-forms": { + "hashes": [ + "sha256:5952bab971110d0b86c278132dae0aa095beee8f723e625c3d3fa28888f1675f", + "sha256:705ededc554ad8736157c666681165fe22ead2dec0d5446d65fc9dd976a5a876" + ], + "index": "pypi", + "version": "==1.7.2" + }, + "django-heroku": { + "hashes": [ + "sha256:2bc690aab89eedbe01311752320a9a12e7548e3b0ed102681acc5736a41a4762", + "sha256:6af4bc3ae4a9b55eaad6dbe5164918982d2762661aebc9f83d9fa49f6009514e" + ], + "index": "pypi", + "version": "==0.3.1" + }, + "django-open-humans": { + "hashes": [ + "sha256:73666363a55937f0ad8cac5ab48f1326822961b154d785005d0bbb4d47fbd4c3" + ], + "index": "pypi", + "version": "==0.1.4" + }, + "gunicorn": { + "hashes": [ + "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", + "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" + ], + "index": "pypi", + "version": "==19.9.0" + }, + "humanfriendly": { + "hashes": [ + "sha256:23057b10ad6f782e7bc3a20e3cb6768ab919f619bbdc0dd75691121bbde5591d", + "sha256:33ee8ceb63f1db61cce8b5c800c531e1a61023ac5488ccde2ba574a85be00a85" + ], + "version": "==4.18" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "kombu": { + "hashes": [ + "sha256:389ba09e03b15b55b1a7371a441c894fd8121d174f5583bbbca032b9ea8c9edd", + "sha256:7b92303af381ef02fad6899fd5f5a9a96031d781356cd8e505fa54ae5ddee181" + ], + "version": "==4.5.0" + }, + "open-humans-api": { + "hashes": [ + "sha256:44d881bd361fd006db8748e64b111e1a5ef0afbe8ea8ad9507fd0e1fa8766306" + ], + "version": "==0.2.5" + }, + "psycopg2": { + "hashes": [ + "sha256:3648afc2b4828a6e00d516d2d09a260edd2c1e3de1e0d41d99c5ab004a73d180", + "sha256:5329b4530e31f58e0eafc55e26bbef684509bcc3be41604e45c0b98c297dc722", + "sha256:7c1ae1669d11105a002f804bebd7432f8dc7473459aa405164c6b44a922decd5", + "sha256:8af13498e32a00d0a66e43b7491c15231b27ab964ee4d2277a4a2dbadfb2c482", + "sha256:9d5489867bd5f6d6c6191a4debd8de9a5c03a9608cce3f4d7133e29e6bd4ec27", + "sha256:a17bfc9faffcca0ad9360c1ad97ab61ede583aa954715e8e436ffd80046661ff", + "sha256:b4a475ce87eabc0607e068a3c704d0aa0820237ed78d493b8e2d880eb73cd7fe", + "sha256:c49d66e97affdc80d084b3b363f09f17db621418f0b8e0524b06c54959e2094d", + "sha256:d13fbc3d533656cfdf094e13c1b0f40917b72813755ba780971ba0ce04280ac4", + "sha256:e1e4fe6e8ab9f9c7d28514d007f623999d2dd6b5b81069dd4f9d30dbdd6f7069", + "sha256:e67d60cb1a32f5fd8fcea935cf9efb1d1c26f96203b0ca2ae98c4c40ef8d8eac" + ], + "version": "==2.8.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "version": "==2.8.0" + }, + "pytz": { + "hashes": [ + "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", + "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + ], + "version": "==2019.1" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "index": "pypi", + "version": "==2.21.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "sqlparse": { + "hashes": [ + "sha256:40afe6b8d4b1117e7dff5504d7a8ce07d9a1b15aeeade8a2d10f130a834f8177", + "sha256:7c3dca29c022744e95b547e867cee89f4fce4373f3549ccd8797d8eb52cdb873" + ], + "version": "==0.3.0" + }, + "urllib3": { + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + }, + "vine": { + "hashes": [ + "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87", + "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af" + ], + "version": "==1.3.0" + }, + "whitenoise": { + "hashes": [ + "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", + "sha256:42133ddd5229eeb6a0c9899496bdbe56c292394bf8666da77deeb27454c0456a" + ], + "version": "==4.1.2" + } + }, + "develop": {} +} diff --git a/loggather/settings.py b/loggather/settings.py index f3784d7..da43495 100644 --- a/loggather/settings.py +++ b/loggather/settings.py @@ -4,14 +4,9 @@ import os -from env_tools import apply_env - import django_heroku -# Apply the env in the .env file -apply_env() - # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -22,7 +17,7 @@ DEBUG = True if os.getenv("DEBUG", "true").lower() == "true" else False -ALLOWED_HOSTS = ["*"] +ALLOWED_HOSTS = [] ROOT_URLCONF = "loggather.urls" @@ -124,7 +119,6 @@ OPENHUMANS_OH_BASE_URL = os.getenv( "OPENHUMANS_OH_BASE_URL", "https://www.openhumans.org" ) -OHAPI_OH_BASE_URL = OPENHUMANS_OH_BASE_URL CELERY_BROKER_URL = os.getenv("REDIS_URL") CELERY_TASK_SERIALIZER = "json" diff --git a/loggather/tasks.py b/loggather/tasks.py index bb35f74..c40177c 100644 --- a/loggather/tasks.py +++ b/loggather/tasks.py @@ -13,7 +13,7 @@ logger = get_task_logger(__name__) -def serialize_accesslogs(api_endpoint, oh_member, access_token, start_date, end_date): +def serialize_accesslogs(api_endpoint, oh_member, start_date, end_date): """ Groups logs by project, then converts from dict to csv, and finally uploads the resultant csv files to aws. @@ -132,14 +132,13 @@ def serialize_accesslogs(api_endpoint, oh_member, access_token, start_date, end_ csv = "{0}\n{1}".format(",".join(headers), csv) # Prepend the headers f = io.StringIO(csv) logger.info("Writing {0}".format(filename)) - upload_stream( + oh_member.upload( f, filename, metadata={ "description": "Open Humans access logs: AWS side", "tags": ["logs", "access logs", "AWS access logs"], }, - access_token=access_token, ) @@ -149,12 +148,11 @@ def get_logs(oh_member_pk, start_date=None, end_date=None): Celery task to retrieve the specified set of logs and save them as files """ oh_member = OpenHumansMember.objects.get(pk=oh_member_pk) - access_token = oh_member.get_access_token() serialize_accesslogs( - "newdatafileaccesslog", oh_member, access_token, start_date, end_date + "newdatafileaccesslog", oh_member, start_date, end_date ) serialize_accesslogs( - "awsdatafileaccesslog", oh_member, access_token, start_date, end_date + "awsdatafileaccesslog", oh_member, start_date, end_date ) return diff --git a/requirements.in b/requirements.in deleted file mode 100644 index dcc5672..0000000 --- a/requirements.in +++ /dev/null @@ -1,12 +0,0 @@ -arrow -celery -django -django-crispy-forms -django-heroku -django-open-humans -env-tools -gunicorn -Markdown -oauthlib -redis -requests \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c98562d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,37 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile requirements.in -# -amqp==2.4.2 # via kombu -arrow==0.13.1 -billiard==3.6.0.0 # via celery -celery==4.3.0 -certifi==2019.3.9 # via requests -chardet==3.0.4 # via requests -click==7.0 # via open-humans-api -configparser==3.7.4 # via tini -django-crispy-forms==1.7.2 -django-heroku==0.3.1 -django-open-humans==0.1.3 -django==2.2 -env-tools==2.2.0 -gunicorn==19.9.0 -humanfriendly==4.18 # via open-humans-api -idna==2.8 # via requests -kombu==4.5.0 # via celery -markdown==3.1 -oauthlib==3.0.1 -open-humans-api==0.2.5 # via django-open-humans -psycopg2==2.7.7 # via django-heroku -python-dateutil==2.8.0 # via arrow -pytz==2019.1 # via celery, django -redis==3.2.1 -requests==2.21.0 -six==1.12.0 # via env-tools, python-dateutil, tini -sqlparse==0.3.0 # via django -tini==3.0.1 # via env-tools -urllib3==1.24.1 # via requests -vine==1.3.0 # via amqp, celery -whitenoise==4.1.2 # via django-heroku From f59258eee1466edbc2aa25b4ae3cb2539834ad93 Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Wed, 10 Apr 2019 15:24:45 -0700 Subject: [PATCH 2/4] Add example .env file --- env.example | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 env.example diff --git a/env.example b/env.example new file mode 100644 index 0000000..e415b61 --- /dev/null +++ b/env.example @@ -0,0 +1,6 @@ +SECRET_KEY='secretgoeshere' +OPENHUMANS_CLIENT_ID='clientidhere' +OPENHUMANS_CLIENT_SECRET='clientsecrethere' + +# Set if you want to test against local version of main site +# OPENHUMANS_OH_BASE_URL='http://127.0.0.1:8000' From e600d09f78dd22398e6c31cbcc4dc571210aaa82 Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Wed, 10 Apr 2019 16:19:03 -0700 Subject: [PATCH 3/4] Add redis to pipfile and default to settings --- Pipfile | 1 + Pipfile.lock | 10 +++++++++- env.example | 3 +++ loggather/settings.py | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 53506c3..b45811d 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,7 @@ django-heroku = "*" django-crispy-forms = "*" requests = "*" django-open-humans = "*" +redis = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index db0aa87..da3b461 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "71500f4f15429165fdb4947060e134501a3fe41dc109ca923b63beeb0d5d7148" + "sha256": "d661df0508c098fa8c1bf65415a64a399c2c37c43c3bbc1b16c9883e19658f31" }, "pipfile-spec": 6, "requires": { @@ -168,6 +168,14 @@ ], "version": "==2019.1" }, + "redis": { + "hashes": [ + "sha256:6946b5dca72e86103edc8033019cc3814c031232d339d5f4533b02ea85685175", + "sha256:8ca418d2ddca1b1a850afa1680a7d2fd1f3322739271de4b704e0d4668449273" + ], + "index": "pypi", + "version": "==3.2.1" + }, "requests": { "hashes": [ "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", diff --git a/env.example b/env.example index e415b61..01df412 100644 --- a/env.example +++ b/env.example @@ -4,3 +4,6 @@ OPENHUMANS_CLIENT_SECRET='clientsecrethere' # Set if you want to test against local version of main site # OPENHUMANS_OH_BASE_URL='http://127.0.0.1:8000' + +# Set if you want another redis URL +# REDIS_URL='redis://' \ No newline at end of file diff --git a/loggather/settings.py b/loggather/settings.py index da43495..e0f4a0c 100644 --- a/loggather/settings.py +++ b/loggather/settings.py @@ -120,7 +120,7 @@ "OPENHUMANS_OH_BASE_URL", "https://www.openhumans.org" ) -CELERY_BROKER_URL = os.getenv("REDIS_URL") +CELERY_BROKER_URL = os.getenv("REDIS_URL", "redis://") CELERY_TASK_SERIALIZER = "json" LOGIN_URL = "/" From 5c9b834435c1d6b58233ca02581ac527f7580248 Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Thu, 11 Apr 2019 09:46:30 -0700 Subject: [PATCH 4/4] Switch Python --- runtime.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 runtime.txt diff --git a/runtime.txt b/runtime.txt deleted file mode 100644 index 6f651a3..0000000 --- a/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.7.3