From e1dca5b9097d35131462c43af93c7124a391a043 Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Mon, 17 Aug 2020 17:18:13 +0200 Subject: [PATCH 1/9] add related settings in env --- docker/.env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index dc137c03..7641da47 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -7,3 +7,6 @@ STATIC_URL=/miller-assets/ MILLER_SCHEMA_ROOT=/contents/schema DEBUG=False LANGUAGES=en|American English|en_US|english,fr|French|fr_FR|french,de|German|de_DE|german +MILLER_CONTENTS_ENABLE_GIT=True +MILLER_CONTENTS_GIT_USERNAME=your-username-for-git +MILLER_CONTENTS_GIT_EMAIL=your-email-for-git From ab6f916da44f5a1cb7894a377811503d5e58abe6 Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Mon, 17 Aug 2020 18:17:36 +0200 Subject: [PATCH 2/9] add git related env variables --- miller/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/miller/settings.py b/miller/settings.py index 5d9ff38b..c2210d5d 100644 --- a/miller/settings.py +++ b/miller/settings.py @@ -226,6 +226,8 @@ MILLER_CONTENTS_ROOT = get_env_variable('CONTENTS_ROOT', '/contents') MILLER_CONTENTS_ENABLE_GIT = get_env_variable( 'MILLER_CONTENTS_ENABLE_GIT', 'True') == 'True' +MILLER_CONTENTS_GIT_USERNAME = get_env_variable('MILLER_CONTENTS_GIT_USERNAME', 'miller') +MILLER_CONTENTS_GIT_EMAIL = get_env_variable('MILLER_CONTENTS_GIT_EMAIL', 'donotreply@miller') # snapshots and thumbnail sizes # default: max size, both heght and width must be 1200 px MILLER_SIZES_SNAPSHOT = [ From 7bac146afdf51610d39e4a6d5806d7bb8c88d33a Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Mon, 17 Aug 2020 18:18:05 +0200 Subject: [PATCH 3/9] add boolean env in docker-compose --- docker/docker-compose.dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index b8fa6a4c..c88010e5 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -34,6 +34,7 @@ services: MILLER_DATABASE_HOST: postgresdb MILLER_DATABASE_PORT: 5432 MILLER_SCHEMA_ROOT: ${MILLER_SCHEMA_ROOT} + MILLER_CONTENTS_ENABLE_GIT: ${MILLER_CONTENTS_ENABLE_GIT} MILLER_GIT_TAG: ${GIT_TAG} MILLER_GIT_BRANCH: ${GIT_BRANCH} MILLER_GIT_REVISION: ${GIT_REVISION} @@ -64,6 +65,7 @@ services: MILLER_DATABASE_HOST: postgresdb MILLER_DATABASE_PORT: 5432 MILLER_SCHEMA_ROOT: ${MILLER_SCHEMA_ROOT} + MILLER_CONTENTS_ENABLE_GIT: ${MILLER_CONTENTS_ENABLE_GIT} MILLER_GIT_TAG: ${GIT_TAG} MILLER_GIT_BRANCH: ${GIT_BRANCH} MILLER_GIT_REVISION: ${GIT_REVISION} From 28c10fca8e4f29e04cac061f4a997b1d9a8bc61d Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Mon, 17 Aug 2020 18:18:29 +0200 Subject: [PATCH 4/9] add utils to commit files --- miller/utils/git.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 miller/utils/git.py diff --git a/miller/utils/git.py b/miller/utils/git.py new file mode 100644 index 00000000..5bfa5fc8 --- /dev/null +++ b/miller/utils/git.py @@ -0,0 +1,33 @@ +import os +from git import Repo, Commit, Actor, Tree +from django.conf import settings + +def get_repo(): + if not settings.MILLER_CONTENTS_ENABLE_GIT: + raise NameError('You shoud enable MILLER_CONTENTS_ENABLE_GIT in the settings file') + if not settings.MILLER_CONTENTS_ROOT: + raise NameError('You shoud set MILLER_CONTENTS_ROOT to an absolute path in the settings file') + if not os.path.exists(settings.MILLER_CONTENTS_ROOT): + raise OsError(f'path for MILLER_CONTENTS_ROOT does not exist or it is not reachable: {settings.MILLER_CONTENTS_ROOT}') + repo = Repo.init(settings.MILLER_CONTENTS_ROOT) + return repo + +def get_or_create_path_in_contents_root(folder_path): + prefixed_path = os.path.join( + settings.MILLER_CONTENTS_ROOT, + folder_path + ) + if not os.path.exists(prefixed_path): + os.makedirs(prefixed_path) + with open(os.path.join(prefixed_path, '.gitkeep'), 'w'): + pass + return prefixed_path + +def commit_filepath(filepath, username, email, message): + author = Actor(username, email) + committer = Actor(username, email) + repo = get_repo() + repo.index.add([filepath]) + commit_message = repo.index.commit(message=message, author=author, committer=committer) + short_sha = repo.git.rev_parse(commit_message, short=7) + return short_sha From d60818cda977489bd3a9ba21556ac881e22c6311 Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Mon, 17 Aug 2020 18:18:58 +0200 Subject: [PATCH 5/9] serialize a document as YAML then commit to contents root if git is enabled --- miller/management/commands/document_commit.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 miller/management/commands/document_commit.py diff --git a/miller/management/commands/document_commit.py b/miller/management/commands/document_commit.py new file mode 100644 index 00000000..9dc33a95 --- /dev/null +++ b/miller/management/commands/document_commit.py @@ -0,0 +1,56 @@ +import os +from django.conf import settings +from django.core import serializers +from django.core.management.base import BaseCommand +from miller.models import Document +from miller.utils.git import get_or_create_path_in_contents_root, commit_filepath + +class Command(BaseCommand): + """ + usage: + ENV=development pipenv run ./manage.py document_commit + or if in docker: + docker exec -it docker_miller_1 \ + python manage.py document_commit \ + , ... [--immediate] [--verbose] + """ + help = 'save YAML version of documents and commit them (if a git repo is enaled)' + + def add_arguments(self, parser): + parser.add_argument('document_pks', nargs='+', type=int) + parser.add_argument( + '--immediate', + action='store_true', + help='avoid delay tasks using celery', + ) + parser.add_argument( + '--verbose', + action='store_true', + help='use verbose logging', + ) + + def handle( + self, document_pks, immediate=False, verbose=False, *args, **options + ): + if not settings.MILLER_CONTENTS_ENABLE_GIT: + self.stderr.write('MILLER_CONTENTS_ENABLE_GIT not enabled!') + # repo = get_repo() + self.stdout.write(f'document_commit for: {document_pks}') + docs = Document.objects.filter(pk__in=document_pks) + folder_path = get_or_create_path_in_contents_root('document') + # loop through documents + for doc in docs: + self.stdout.write(f'document: {doc.pk} {doc.short_url}') + contents = serializers.serialize('yaml', [doc]) + filepath = os.path.join(folder_path, f'{doc.pk}-{doc.short_url}.yml') + with open(filepath, 'w', encoding='utf-8') as f: + f.write(contents) + self.stdout.write(f'document: {doc.pk} {doc.short_url} written to {filepath}') + if settings.MILLER_CONTENTS_ENABLE_GIT: + short_sha = commit_filepath( + filepath=filepath, + username=doc.owner if doc.owner is not None else settings.MILLER_CONTENTS_GIT_USERNAME, + email=settings.MILLER_CONTENTS_GIT_EMAIL, + message=f'saving {doc.title}' + ) + self.stdout.write(f'document: {doc.pk} {doc.short_url} committed: {short_sha}') From 0520872057c8092cb4360b45694b714b6e72dd00 Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Tue, 18 Aug 2020 11:19:52 +0200 Subject: [PATCH 6/9] add commands for stories and documents --- miller/management/commands/document_commit.py | 26 ++++------- miller/management/commands/story_commit.py | 46 +++++++++++++++++++ 2 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 miller/management/commands/story_commit.py diff --git a/miller/management/commands/document_commit.py b/miller/management/commands/document_commit.py index 9dc33a95..2138a0ca 100644 --- a/miller/management/commands/document_commit.py +++ b/miller/management/commands/document_commit.py @@ -1,9 +1,7 @@ -import os from django.conf import settings -from django.core import serializers from django.core.management.base import BaseCommand from miller.models import Document -from miller.utils.git import get_or_create_path_in_contents_root, commit_filepath +from miller.tasks import commit_document class Command(BaseCommand): """ @@ -37,20 +35,12 @@ def handle( # repo = get_repo() self.stdout.write(f'document_commit for: {document_pks}') docs = Document.objects.filter(pk__in=document_pks) - folder_path = get_or_create_path_in_contents_root('document') # loop through documents for doc in docs: - self.stdout.write(f'document: {doc.pk} {doc.short_url}') - contents = serializers.serialize('yaml', [doc]) - filepath = os.path.join(folder_path, f'{doc.pk}-{doc.short_url}.yml') - with open(filepath, 'w', encoding='utf-8') as f: - f.write(contents) - self.stdout.write(f'document: {doc.pk} {doc.short_url} written to {filepath}') - if settings.MILLER_CONTENTS_ENABLE_GIT: - short_sha = commit_filepath( - filepath=filepath, - username=doc.owner if doc.owner is not None else settings.MILLER_CONTENTS_GIT_USERNAME, - email=settings.MILLER_CONTENTS_GIT_EMAIL, - message=f'saving {doc.title}' - ) - self.stdout.write(f'document: {doc.pk} {doc.short_url} committed: {short_sha}') + try: + if immediate: + doc.commit() + else: + commit_document.delay(document_pk=doc.pk) + except Exception as e: + self.stderr.write(e) diff --git a/miller/management/commands/story_commit.py b/miller/management/commands/story_commit.py new file mode 100644 index 00000000..385fd9d3 --- /dev/null +++ b/miller/management/commands/story_commit.py @@ -0,0 +1,46 @@ +from django.conf import settings +from django.core.management.base import BaseCommand +from miller.models import Story +from miller.tasks import commit_story + +class Command(BaseCommand): + """ + usage: + ENV=development pipenv run ./manage.py story_commit + or if in docker: + docker exec -it docker_miller_1 \ + python manage.py story_commit \ + , ... [--immediate] [--verbose] + """ + help = 'save YAML version of documents and commit them (if a git repo is enaled)' + + def add_arguments(self, parser): + parser.add_argument('story_pks', nargs='+', type=int) + parser.add_argument( + '--immediate', + action='store_true', + help='avoid delay tasks using celery', + ) + parser.add_argument( + '--verbose', + action='store_true', + help='use verbose logging', + ) + + def handle( + self, story_pks, immediate=False, verbose=False, *args, **options + ): + if not settings.MILLER_CONTENTS_ENABLE_GIT: + self.stderr.write('MILLER_CONTENTS_ENABLE_GIT not enabled!') + # repo = get_repo() + self.stdout.write(f'story_commit for: {story_pks}') + docs = Story.objects.filter(pk__in=story_pks) + # loop through documents + for doc in docs: + try: + if immediate: + doc.commit() + else: + commit_story.delay(story_pk=doc.pk) + except Exception as e: + self.stderr.write(e) From 69465ddfcc5827819ba4c9e8f90cb2036ef63cf8 Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Tue, 18 Aug 2020 11:20:03 +0200 Subject: [PATCH 7/9] add commit function in models --- miller/models/document.py | 7 ++++++- miller/models/story.py | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/miller/models/document.py b/miller/models/document.py index 643d17dc..d8d03551 100644 --- a/miller/models/document.py +++ b/miller/models/document.py @@ -9,6 +9,7 @@ from ..fields import UTF8JSONField from ..snapshots import create_snapshot, create_different_sizes_from_snapshot from ..utils.models import get_search_vector_query, create_short_url +from ..utils.git import commit_instance from ..utils.media import get_video_subtitles @@ -271,4 +272,8 @@ def update_search_vector(self, verbose=False): for value, w, c in contents ] + [self.pk]) - \ No newline at end of file + def commit(self): + """ + if settings.MILLER_CONTENTS_ENABLE_GIT, write to disk + """ + commit_instance(instance=self) diff --git a/miller/models/story.py b/miller/models/story.py index ae5797f6..0055d32b 100644 --- a/miller/models/story.py +++ b/miller/models/story.py @@ -9,6 +9,7 @@ from . import Tag from ..utils import get_all_values_from_dict_by_key from ..utils.models import get_user_path, create_short_url, get_unique_slug +from ..utils.git import commit_instance from ..fields import UTF8JSONField @@ -93,7 +94,7 @@ class Story(models.Model): # add huge search field search_vector = SearchVectorField(null=True, blank=True) - + # enable full text search using postgres vectors stored in search_vector allow_fulltext_search = True @@ -152,3 +153,9 @@ def save_captions_from_contents(self, key='pk', parser='json'): # save captions saved = ThroughModel.objects.bulk_create([ThroughModel(document=d, story=self) for d in docs]) return saved, missing, expecting + + def commit(self): + """ + if settings.MILLER_CONTENTS_ENABLE_GIT, write to disk + """ + commit_instance(instance=self) From 8a00ff5c91c8c0c17e1aa51d2d224c992c651bcb Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Tue, 18 Aug 2020 11:20:28 +0200 Subject: [PATCH 8/9] add commit task for document and story --- miller/tasks.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/miller/tasks.py b/miller/tasks.py index a28da7b0..aabcae09 100644 --- a/miller/tasks.py +++ b/miller/tasks.py @@ -38,6 +38,32 @@ def update_document_search_vectors(self, document_pk, verbose=False): ) +@app.task( + bind=True, autoretry_for=(Exception,), exponential_backoff=2, + retry_kwargs={'max_retries': 5}, retry_jitter=True +) +def commit_document(self, document_pk, verbose=False): + logger.info(f'commit_document document(pk={document_pk})') + doc = Document.objects.get(pk=document_pk) + doc.commit() + logger.info( + f'commit_document document(pk={document_pk}) success.' + ) + + +@app.task( + bind=True, autoretry_for=(Exception,), exponential_backoff=2, + retry_kwargs={'max_retries': 5}, retry_jitter=True +) +def commit_story(self, story_pk, verbose=False): + logger.info(f'commit_story document(pk={story_pk})') + story = Story.objects.get(pk=story_pk) + story.commit() + logger.info( + f'commit_story document(pk={story_pk}) success.' + ) + + @app.task( bind=True, autoretry_for=(Exception,), exponential_backoff=2, retry_kwargs={'max_retries': 5}, retry_jitter=True @@ -62,4 +88,3 @@ def update_document_data_by_type(self, document_pk): def document_post_save_handler(sender, instance, **kwargs): logger.info(f'received @post_save document_pk: {instance.pk}') update_document_search_vectors.delay(document_pk=instance.pk) - From fa7a3ca41a36e7a85007de13f1bc50b3e3304765 Mon Sep 17 00:00:00 2001 From: Daniele Guido Date: Tue, 18 Aug 2020 11:21:24 +0200 Subject: [PATCH 9/9] add commit_instance helper (for every model that can be serialized) --- miller/utils/git.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/miller/utils/git.py b/miller/utils/git.py index 5bfa5fc8..3192e8d5 100644 --- a/miller/utils/git.py +++ b/miller/utils/git.py @@ -1,6 +1,10 @@ import os -from git import Repo, Commit, Actor, Tree +import logging +from git import Repo, Actor from django.conf import settings +from django.core import serializers + +logger = logging.getLogger(__name__) def get_repo(): if not settings.MILLER_CONTENTS_ENABLE_GIT: @@ -8,7 +12,7 @@ def get_repo(): if not settings.MILLER_CONTENTS_ROOT: raise NameError('You shoud set MILLER_CONTENTS_ROOT to an absolute path in the settings file') if not os.path.exists(settings.MILLER_CONTENTS_ROOT): - raise OsError(f'path for MILLER_CONTENTS_ROOT does not exist or it is not reachable: {settings.MILLER_CONTENTS_ROOT}') + raise OsError(f'path for MILLER_CONTENTS_ROOT does not exist or it is not reachable: {settings.MILLER_CONTENTS_ROOT}') # noqa: F821 repo = Repo.init(settings.MILLER_CONTENTS_ROOT) return repo @@ -31,3 +35,27 @@ def commit_filepath(filepath, username, email, message): commit_message = repo.index.commit(message=message, author=author, committer=committer) short_sha = repo.git.rev_parse(commit_message, short=7) return short_sha + + +def commit_instance( + instance, + verbose=False, + serializer='yaml' +): + logger.info( + f'commit_instance for instance pk:{instance.pk} model:{instance._meta.model.__name__}' + ) + folder_path = get_or_create_path_in_contents_root(folder_path=instance._meta.model.__name__) + contents = serializers.serialize(serializer, [instance]) + filepath = os.path.join(folder_path, f'{instance.pk}-{instance.short_url}.yml') + with open(filepath, 'w', encoding='utf-8') as f: + f.write(contents) + logger.info(f'commit_instance document: {instance.pk} {instance.short_url} written to {filepath}') + if settings.MILLER_CONTENTS_ENABLE_GIT: + short_sha = commit_filepath( + filepath=filepath, + username=instance.owner if instance.owner is not None else settings.MILLER_CONTENTS_GIT_USERNAME, + email=settings.MILLER_CONTENTS_GIT_EMAIL, + message=f'saving {instance.title}' + ) + logger.info(f'commit_instance document: {instance.pk} {instance.short_url} committed: {short_sha}')