Feature/yaml in git

docker/.env.example

@@ -7,3 +7,6 @@ STATIC_URL=/miller-assets/
 MILLER_SCHEMA_ROOT=/contents/schema
 DEBUG=False
 LANGUAGES=en|American English|en_US|english,fr|French|fr_FR|french,de|German|de_DE|german
+MILLER_CONTENTS_ENABLE_GIT=True
+MILLER_CONTENTS_GIT_USERNAME=your-username-for-git
+MILLER_CONTENTS_GIT_EMAIL=your-email-for-git

docker/docker-compose.dev.yml

@@ -34,6 +34,7 @@ services:
       MILLER_DATABASE_HOST: postgresdb
       MILLER_DATABASE_PORT: 5432
       MILLER_SCHEMA_ROOT: ${MILLER_SCHEMA_ROOT}
+      MILLER_CONTENTS_ENABLE_GIT: ${MILLER_CONTENTS_ENABLE_GIT}
       MILLER_GIT_TAG: ${GIT_TAG}
       MILLER_GIT_BRANCH: ${GIT_BRANCH}
       MILLER_GIT_REVISION: ${GIT_REVISION}
@@ -64,6 +65,7 @@ services:
       MILLER_DATABASE_HOST: postgresdb
       MILLER_DATABASE_PORT: 5432
       MILLER_SCHEMA_ROOT: ${MILLER_SCHEMA_ROOT}
+      MILLER_CONTENTS_ENABLE_GIT: ${MILLER_CONTENTS_ENABLE_GIT}
       MILLER_GIT_TAG: ${GIT_TAG}
       MILLER_GIT_BRANCH: ${GIT_BRANCH}
       MILLER_GIT_REVISION: ${GIT_REVISION}

miller/management/commands/document_commit.py

@@ -0,0 +1,46 @@
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from miller.models import Document
+from miller.tasks import commit_document
+
+class Command(BaseCommand):
+    """
+    usage:
+    ENV=development pipenv run ./manage.py document_commit <pks>
+    or if in docker:
+    docker exec -it docker_miller_1 \
+    python manage.py document_commit \
+    <document_pk>, <document_pk> ... [--immediate] [--verbose]
+    """
+    help = 'save YAML version of documents and commit them (if a git repo is enaled)'
+
+    def add_arguments(self, parser):
+        parser.add_argument('document_pks', nargs='+', type=int)
+        parser.add_argument(
+            '--immediate',
+            action='store_true',
+            help='avoid delay tasks using celery',
+        )
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='use verbose logging',
+        )
+
+    def handle(
+        self, document_pks, immediate=False, verbose=False, *args, **options
+    ):
+        if not settings.MILLER_CONTENTS_ENABLE_GIT:
+            self.stderr.write('MILLER_CONTENTS_ENABLE_GIT not enabled!')
+        # repo = get_repo()
+        self.stdout.write(f'document_commit for: {document_pks}')
+        docs = Document.objects.filter(pk__in=document_pks)
+        # loop through documents
+        for doc in docs:
+            try:
+                if immediate:
+                    doc.commit()
+                else:
+                    commit_document.delay(document_pk=doc.pk)
+            except Exception as e:
+                self.stderr.write(e)

miller/management/commands/story_commit.py

@@ -0,0 +1,46 @@
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from miller.models import Story
+from miller.tasks import commit_story
+
+class Command(BaseCommand):
+    """
+    usage:
+    ENV=development pipenv run ./manage.py story_commit <pks>
+    or if in docker:
+    docker exec -it docker_miller_1 \
+    python manage.py story_commit \
+    <story_pk>, <story_pk> ... [--immediate] [--verbose]
+    """
+    help = 'save YAML version of documents and commit them (if a git repo is enaled)'
+
+    def add_arguments(self, parser):
+        parser.add_argument('story_pks', nargs='+', type=int)
+        parser.add_argument(
+            '--immediate',
+            action='store_true',
+            help='avoid delay tasks using celery',
+        )
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='use verbose logging',
+        )
+
+    def handle(
+        self, story_pks, immediate=False, verbose=False, *args, **options
+    ):
+        if not settings.MILLER_CONTENTS_ENABLE_GIT:
+            self.stderr.write('MILLER_CONTENTS_ENABLE_GIT not enabled!')
+        # repo = get_repo()
+        self.stdout.write(f'story_commit for: {story_pks}')
+        docs = Story.objects.filter(pk__in=story_pks)
+        # loop through documents
+        for doc in docs:
+            try:
+                if immediate:
+                    doc.commit()
+                else:
+                    commit_story.delay(story_pk=doc.pk)
+            except Exception as e:
+                self.stderr.write(e)

miller/models/document.py

@@ -9,6 +9,7 @@
 from ..fields import UTF8JSONField
 from ..snapshots import create_snapshot, create_different_sizes_from_snapshot
 from ..utils.models import get_search_vector_query, create_short_url
+from ..utils.git import commit_instance
 from ..utils.media import get_video_subtitles
 
 
@@ -271,4 +272,8 @@ def update_search_vector(self, verbose=False):
                 for value, w, c in contents
             ] + [self.pk])
 
-
+    def commit(self):
+        """
+        if settings.MILLER_CONTENTS_ENABLE_GIT, write to disk
+        """
+        commit_instance(instance=self)

miller/models/story.py

@@ -9,6 +9,7 @@
 from . import Tag
 from ..utils import get_all_values_from_dict_by_key
 from ..utils.models import get_user_path, create_short_url, get_unique_slug
+from ..utils.git import commit_instance
 from ..fields import UTF8JSONField
 
 
@@ -93,7 +94,7 @@ class Story(models.Model):
 
     # add huge search field
     search_vector = SearchVectorField(null=True, blank=True)
-    
+
     # enable full text search using postgres vectors stored in search_vector
     allow_fulltext_search = True
 
@@ -152,3 +153,9 @@ def save_captions_from_contents(self, key='pk', parser='json'):
         # save captions
         saved = ThroughModel.objects.bulk_create([ThroughModel(document=d, story=self) for d in docs])
         return saved, missing, expecting
+
+    def commit(self):
+        """
+        if settings.MILLER_CONTENTS_ENABLE_GIT, write to disk
+        """
+        commit_instance(instance=self)

miller/settings.py

@@ -226,6 +226,8 @@
 MILLER_CONTENTS_ROOT = get_env_variable('CONTENTS_ROOT', '/contents')
 MILLER_CONTENTS_ENABLE_GIT = get_env_variable(
     'MILLER_CONTENTS_ENABLE_GIT', 'True') == 'True'
+MILLER_CONTENTS_GIT_USERNAME = get_env_variable('MILLER_CONTENTS_GIT_USERNAME', 'miller')
+MILLER_CONTENTS_GIT_EMAIL = get_env_variable('MILLER_CONTENTS_GIT_EMAIL', 'donotreply@miller')
 # snapshots and thumbnail sizes
 # default: max size, both heght and width must be 1200 px
 MILLER_SIZES_SNAPSHOT = [

miller/tasks.py

@@ -38,6 +38,32 @@ def update_document_search_vectors(self, document_pk, verbose=False):
     )
 
 
+@app.task(
+    bind=True, autoretry_for=(Exception,), exponential_backoff=2,
+    retry_kwargs={'max_retries': 5}, retry_jitter=True
+)
+def commit_document(self, document_pk, verbose=False):
+    logger.info(f'commit_document document(pk={document_pk})')
+    doc = Document.objects.get(pk=document_pk)
+    doc.commit()
+    logger.info(
+        f'commit_document document(pk={document_pk}) success.'
+    )
+
+
+@app.task(
+    bind=True, autoretry_for=(Exception,), exponential_backoff=2,
+    retry_kwargs={'max_retries': 5}, retry_jitter=True
+)
+def commit_story(self, story_pk, verbose=False):
+    logger.info(f'commit_story document(pk={story_pk})')
+    story = Story.objects.get(pk=story_pk)
+    story.commit()
+    logger.info(
+        f'commit_story document(pk={story_pk}) success.'
+    )
+
+
 @app.task(
     bind=True, autoretry_for=(Exception,), exponential_backoff=2,
     retry_kwargs={'max_retries': 5}, retry_jitter=True
@@ -62,4 +88,3 @@ def update_document_data_by_type(self, document_pk):
 def document_post_save_handler(sender, instance, **kwargs):
     logger.info(f'received @post_save document_pk: {instance.pk}')
     update_document_search_vectors.delay(document_pk=instance.pk)
-

miller/utils/git.py

@@ -0,0 +1,61 @@
+import os
+import logging
+from git import Repo, Actor
+from django.conf import settings
+from django.core import serializers
+
+logger = logging.getLogger(__name__)
+
+def get_repo():
+    if not settings.MILLER_CONTENTS_ENABLE_GIT:
+        raise NameError('You shoud enable MILLER_CONTENTS_ENABLE_GIT in the settings file')
+    if not settings.MILLER_CONTENTS_ROOT:
+        raise NameError('You shoud set MILLER_CONTENTS_ROOT to an absolute path in the settings file')
+    if not os.path.exists(settings.MILLER_CONTENTS_ROOT):
+        raise OsError(f'path for MILLER_CONTENTS_ROOT does not exist or it is not reachable: {settings.MILLER_CONTENTS_ROOT}')  # noqa: F821
+    repo = Repo.init(settings.MILLER_CONTENTS_ROOT)
+    return repo
+
+def get_or_create_path_in_contents_root(folder_path):
+    prefixed_path = os.path.join(
+        settings.MILLER_CONTENTS_ROOT,
+        folder_path
+    )
+    if not os.path.exists(prefixed_path):
+        os.makedirs(prefixed_path)
+    with open(os.path.join(prefixed_path, '.gitkeep'), 'w'):
+        pass
+    return prefixed_path
+
+def commit_filepath(filepath, username, email, message):
+    author = Actor(username, email)
+    committer = Actor(username, email)
+    repo = get_repo()
+    repo.index.add([filepath])
+    commit_message = repo.index.commit(message=message, author=author, committer=committer)
+    short_sha = repo.git.rev_parse(commit_message, short=7)
+    return short_sha
+
+
+def commit_instance(
+    instance,
+    verbose=False,
+    serializer='yaml'
+):
+    logger.info(
+        f'commit_instance for instance pk:{instance.pk} model:{instance._meta.model.__name__}'
+    )
+    folder_path = get_or_create_path_in_contents_root(folder_path=instance._meta.model.__name__)
+    contents = serializers.serialize(serializer, [instance])
+    filepath = os.path.join(folder_path, f'{instance.pk}-{instance.short_url}.yml')
+    with open(filepath, 'w', encoding='utf-8') as f:
+        f.write(contents)
+        logger.info(f'commit_instance document: {instance.pk} {instance.short_url} written to {filepath}')
+    if settings.MILLER_CONTENTS_ENABLE_GIT:
+        short_sha = commit_filepath(
+            filepath=filepath,
+            username=instance.owner if instance.owner is not None else settings.MILLER_CONTENTS_GIT_USERNAME,
+            email=settings.MILLER_CONTENTS_GIT_EMAIL,
+            message=f'saving {instance.title}'
+        )
+        logger.info(f'commit_instance document: {instance.pk} {instance.short_url} committed: {short_sha}')