diff --git a/README.rst b/README.rst index 901145c..f4f1b34 100644 --- a/README.rst +++ b/README.rst @@ -2,16 +2,13 @@ Complexity ========== -.. image:: https://badge.fury.io/py/complexity.png - :target: http://badge.fury.io/py/complexity - -.. image:: https://travis-ci.org/audreyr/complexity.png?branch=master - :target: https://travis-ci.org/audreyr/complexity +A refreshingly simple static site generator, for those who like to work in HTML. -.. image:: https://pypip.in/d/complexity/badge.png - :target: https://crate.io/packages/complexity?version=latest +Changes +------- -A refreshingly simple static site generator, for those who like to work in HTML. +- New url argument that allows you to (--watch) watch a folder for changes, and any changes will fire off complexity +- New config variable to turn on/off the auto-expand system Documentation ------------- @@ -69,3 +66,4 @@ Community * We love contributions. Read about `how to contribute`_. .. _`how to contribute`: https://github.com/audreyr/complexity/blob/master/CONTRIBUTING.rst + diff --git a/complexity/conf.py b/complexity/conf.py index 252b40f..8aeafbd 100755 --- a/complexity/conf.py +++ b/complexity/conf.py @@ -14,6 +14,16 @@ import yaml +DEFAULTS = { + "templates_dir": "templates/", + "assets_dir": "assets/", + "context_dir": "context/", + "output_dir": "../www/", + "macro_dirs": ["macros/"], + "expand": True +} + + def read_conf(directory): """ Reads and parses the `complexity.yml` configuration file from a diff --git a/complexity/generate.py b/complexity/generate.py index 720699b..6c934b7 100755 --- a/complexity/generate.py +++ b/complexity/generate.py @@ -10,10 +10,11 @@ import json import logging -import os +import os.path import shutil import re +from yaml import safe_load from binaryornot.check import is_binary from jinja2 import FileSystemLoader from jinja2.environment import Environment @@ -22,7 +23,7 @@ from .utils import make_sure_path_exists, unicode_open -def get_output_filename(template_filepath, output_dir, force_unexpanded): +def get_output_filename(template_filepath, output_dir, force_unexpanded, expand): """ Given an input filename, return the corresponding output filename. @@ -41,7 +42,7 @@ def get_output_filename(template_filepath, output_dir, force_unexpanded): if basename.startswith('base'): return False # Put index and unexpanded templates in the root. - elif force_unexpanded or basename == 'index.html': + elif force_unexpanded or basename == 'index.html' or not expand: output_filename = os.path.join(output_dir, template_filepath) # Put other pages in page/index.html, for better URL formatting. else: @@ -68,7 +69,7 @@ def minify_html(html): def generate_html_file(template_filepath, output_dir, env, - context, force_unexpanded=False, minify=False): + context, force_unexpanded=False, minify=False, expand=True): """ Renders and writes a single HTML file to its corresponding output location. @@ -79,6 +80,8 @@ def generate_html_file(template_filepath, :param env: Jinja2 environment with a loader already set up. :param context: Jinja2 context that holds template variables. See http://jinja.pocoo.org/docs/api/#the-context + :param expand: Shall we expand the filenames to folder/index.html + as pretty URLS? """ # Ignore templates starting with "base". They're treated as special cases. @@ -96,7 +99,7 @@ def generate_html_file(template_filepath, rendered_html = minify_html(rendered_html) output_filename = get_output_filename(template_filepath, - output_dir, force_unexpanded) + output_dir, force_unexpanded, expand) if output_filename: make_sure_path_exists(os.path.dirname(output_filename)) @@ -106,8 +109,20 @@ def generate_html_file(template_filepath, return True -def generate_html(templates_dir, output_dir, context=None, - unexpanded_templates=()): +def _ignore(path): + fn = os.path.basename(path) + _, ext = os.path.splitext(path) + if is_binary(path): + return True + if fn == 'complexity.yml': + return True + if ext in ('.j2','.yml'): + return True + return False + + +def generate_html(templates_dir, macro_dirs, output_dir, context=None, + unexpanded_templates=(), expand=True, quiet=False): """ Renders the HTML templates from `templates_dir`, and writes them to `output_dir`. @@ -119,6 +134,9 @@ def generate_html(templates_dir, output_dir, context=None, :paramtype output_dir: directory :param context: Jinja2 context that holds template variables. See http://jinja.pocoo.org/docs/api/#the-context + :param expand: Shall we expand the filenames to folder/index.html + as pretty URLS? + :param quiet: show no output! """ logging.debug('Templates dir is {0}'.format(templates_dir)) @@ -129,9 +147,11 @@ def generate_html(templates_dir, output_dir, context=None, ) context = context or {} - env = Environment() - # os.chdir(templates_dir) - env.loader = FileSystemLoader(templates_dir) + + _dirs = [templates_dir] + _dirs.extend(macro_dirs) + + env = Environment(loader=FileSystemLoader(_dirs)) # Create the output dir if it doesn't already exist make_sure_path_exists(output_dir) @@ -151,15 +171,17 @@ def generate_html(templates_dir, output_dir, context=None, force_unexpanded )) - if is_binary(os.path.join(templates_dir, template_filepath)): - print('Non-text file found: {0}. Skipping.'. - format(template_filepath)) + if _ignore(os.path.join(templates_dir, template_filepath)): + if quiet == False: + print('Ignore: {0}. Skipping.'. + format(template_filepath)) else: outfile = get_output_filename(template_filepath, output_dir, - force_unexpanded) - print('Copying {0} to {1}'.format(template_filepath, outfile)) + force_unexpanded, expand) + if quiet == False: + print('Copying {0} to {1}'.format(template_filepath, outfile)) generate_html_file(template_filepath, output_dir, env, context, - force_unexpanded) + force_unexpanded, expand) def generate_context(context_dir): @@ -194,24 +216,27 @@ def generate_context(context_dir): """ context = {} - json_files = os.listdir(context_dir) - - for file_name in json_files: - - if file_name.endswith('json'): + all_files = os.listdir(context_dir) + for fn in all_files: + path = os.path.join(context_dir, fn) + name, ext = os.path.splitext(fn) - # Open the JSON file and convert to Python object - json_file = os.path.join(context_dir, file_name) - with unicode_open(json_file) as f: + obj = None + if ext == '.json': + with unicode_open(path) as f: obj = json.load(f) + elif ext in {'.yml', '.yaml'}: + with unicode_open(path) as f: + obj = safe_load(f) - # Add the Python object to the context dictionary - context[file_name[:-5]] = obj + if obj is not None: + print('Parsed {0} to context as {1}'.format(fn, name)) + context[name] = obj return context -def copy_assets(assets_dir, output_dir): +def copy_assets(assets_dir, output_dir, quiet=False): """ Copies static assets over from `assets_dir` to `output_dir`. @@ -220,6 +245,7 @@ def copy_assets(assets_dir, output_dir): :paramtype assets_dir: directory :param output_dir: The Complexity output directory, e.g. `www/`. :paramtype output_dir: directory + :param quiet: output to user """ assets = os.listdir(assets_dir) @@ -229,11 +255,13 @@ def copy_assets(assets_dir, output_dir): # Only copy allowed dirs if os.path.isdir(item_path) and item != 'scss' and item != 'less': new_dir = os.path.join(output_dir, item) - print('Copying directory {0} to {1}'.format(item, new_dir)) + if quiet == False: + print('Copying directory {0} to {1}'.format(item, new_dir)) shutil.copytree(item_path, new_dir) # Copy over files in the root of assets_dir if os.path.isfile(item_path): new_file = os.path.join(output_dir, item) - print('Copying file {0} to {1}'.format(item, new_file)) + if quiet == False: + print('Copying file {0} to {1}'.format(item, new_file)) shutil.copyfile(item_path, new_file) diff --git a/complexity/main.py b/complexity/main.py index f871c84..c3e2b09 100755 --- a/complexity/main.py +++ b/complexity/main.py @@ -15,18 +15,35 @@ import logging import os import sys +import time +import json -from .conf import read_conf, get_unexpanded_list +from .conf import read_conf, get_unexpanded_list, DEFAULTS from .exceptions import OutputDirExistsException from .generate import generate_context, copy_assets, generate_html -from .prep import prompt_and_delete_cruft +from .prep import prompt_and_delete_cruft, delete_cruft from .serve import serve_static_site +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler +from time import gmtime, strftime logger = logging.getLogger(__name__) -def complexity(project_dir, no_input=True): +def _get_dir(project_dir, conf_dir_name): + conf_dict = read_conf(project_dir) or DEFAULTS + output_dir = os.path.normpath( + os.path.join(project_dir, conf_dict[conf_dir_name]) + ) + return output_dir + + +def _get_output_dir(project_dir): + return _get_dir(project_dir, 'output_dir') + + +def complexity(project_dir, overwrite=False, no_input=True, quiet=False, settings_json=None, _leave_output=False): """ API equivalent to using complexity at the command line. @@ -40,24 +57,21 @@ def complexity(project_dir, no_input=True): .. note:: You must delete `output_dir` before calling this. This also does not start the Complexity development server; you can do that from your code if desired. + + :para quiet: if True, we won't alert the end user to anything, and we'll + `just run` until we are finished """ # Get the configuration dictionary, if config exists - defaults = { - "templates_dir": "templates/", - "assets_dir": "assets/", - "context_dir": "context/", - "output_dir": "../www/" - } - conf_dict = read_conf(project_dir) or defaults + conf_dict = read_conf(project_dir) or DEFAULTS - output_dir = os.path.normpath( - os.path.join(project_dir, conf_dict['output_dir']) - ) + output_dir = _get_output_dir(project_dir) - # If output_dir exists, prompt before deleting. - # Abort if it can't be deleted. - if no_input: + if overwrite: + delete_cruft(output_dir, only_contents=_leave_output) + elif no_input: + # If output_dir exists, prompt before deleting. + # Abort if it can't be deleted. if os.path.exists(output_dir): raise OutputDirExistsException( 'Please delete {0} manually and try again.' @@ -67,20 +81,40 @@ def complexity(project_dir, no_input=True): sys.exit() # Generate the context data - context = None + context = {} if 'context_dir' in conf_dict: context_dir = os.path.join(project_dir, conf_dict['context_dir']) if os.path.exists(context_dir): - context = generate_context(context_dir) + context.update(generate_context(context_dir)) + + # update the context with anything in the project conf + context.update(conf_dict.get('context', {})) + + # json settings comes last + if settings_json: + # noinspection PyBroadException + try: + settings = json.loads(settings_json) + if isinstance(settings, dict) and settings: + if 'settings' in context: + context['settings'].update(settings) + else: + context['settings'] = settings + print("Updated settings from command-line") + except Exception as e: + pass # Generate and serve the HTML site unexpanded_templates = get_unexpanded_list(conf_dict) templates_dir = os.path.join(project_dir, conf_dict['templates_dir']) - generate_html(templates_dir, output_dir, context, unexpanded_templates) + macro_dirs = [os.path.join(project_dir, _dir) for _dir in conf_dict['macro_dirs']] + + generate_html(templates_dir, macro_dirs, output_dir, + context, unexpanded_templates, conf_dict['expand'], quiet) if 'assets_dir' in conf_dict: assets_dir = os.path.join(project_dir, conf_dict['assets_dir']) - copy_assets(assets_dir, output_dir) + copy_assets(assets_dir, output_dir, quiet) return output_dir @@ -106,24 +140,110 @@ def get_complexity_args(): default=9090, help='Port number to serve files on.' ) + parser.add_argument( + '--address', + default='127.0.0.1', + help='IP to serve files on.' + ) parser.add_argument( '--noserver', action='store_true', help='Don\'t run the server.' ) + parser.add_argument( + '--overwrite', + default=False, + action='store_true', + help='Overwrite the output directory without prompting.' + ) + parser.add_argument( + '--watch', + action='store_true', + help='Will watch a folder for changes and then process if an event is fired' + ) + parser.add_argument( + '--settings', + type=str, default='{}', + help='JSON settings to apply (update) to the loaded context' + ) args = parser.parse_args() return args +def watching_file_system(): + """ + Using watchdog, we'll monitor the filesystem for any changes, and if + we find any, we'll serve the output again (by running complexity) + """ + # Get the path we'll need to monitor, it'll be part of the arg list + args = get_complexity_args() + + # make absolute because server chdirs + proj_dir = os.path.abspath(args.project_dir) + + # Lets observe the folder, and notify complexity when something bad happens + observer = Observer() + event_handler = MyHandler(project_dir=proj_dir) + + paths = [] + for _dir in ("templates_dir", "assets_dir", "context_dir"): + paths.append(_get_dir(proj_dir, _dir)) + + # from _get_dir above + cd = read_conf(proj_dir) or DEFAULTS + for _dir in cd['macro_dirs']: + _path = os.path.normpath(os.path.join(proj_dir, _dir)) + paths.append(_path) + + for path in paths: + print("Watching folder " + path + " for changes:") + observer.schedule(event_handler, path, recursive=True) + observer.start() + + # We'll now continue to look until we Ctrl-C finish + try: + if args.noserver: + while True: + time.sleep(1) + else: + output_dir = _get_output_dir(args.project_dir) + serve_static_site(output_dir=output_dir, address=args.address, port=args.port) + except KeyboardInterrupt: + observer.stop(); + + observer.join() + +""" +This class handles at which points we should process the complexity system again. +We are targeting any events +""" +class MyHandler(FileSystemEventHandler): + + def __init__(self, project_dir, *args, **kwargs): + super(MyHandler, self).__init__(*args, **kwargs) + self._project_dir = project_dir + + def on_any_event(self, event): + args = get_complexity_args() + complexity(project_dir=self._project_dir, no_input=False, + quiet=True, + overwrite=True, + settings_json=args.settings, + _leave_output=True) # delete contents of www + print(" [" + strftime("%Y-%m-%d %H:%M:%S", gmtime()) + "] -> Completed") + def main(): """ Entry point for the package, as defined in `setup.py`. """ args = get_complexity_args() - output_dir = complexity(project_dir=args.project_dir, no_input=False) - if not args.noserver: - serve_static_site(output_dir=output_dir, port=args.port) - + if args.watch == True: + watching_file_system() + else: + output_dir = complexity(project_dir=args.project_dir, overwrite=args.overwrite, no_input=False, + settings_json=args.settings) + if not args.noserver: + serve_static_site(output_dir=output_dir, address=args.address, port=args.port) if __name__ == '__main__': - main() + watching_file_system() diff --git a/complexity/prep.py b/complexity/prep.py index c5c6f8c..cc9bf9d 100755 --- a/complexity/prep.py +++ b/complexity/prep.py @@ -11,10 +11,28 @@ import os import shutil +import glob from . import utils +def delete_cruft(output_dir, only_contents=False): + if only_contents: + print('Deleting {0}/*'.format(output_dir)) + for f in os.listdir(output_dir): + file_path = os.path.join(output_dir, f) + try: + if os.path.isfile(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path, ignore_errors=True) + except Exception as e: + print('Error Deleting') + raise + else: + print('Deleting {0}/*'.format(output_dir)) + shutil.rmtree(output_dir, ignore_errors=True) + def prompt_and_delete_cruft(output_dir): """ Asks if it's okay to delete `output_dir/`. @@ -38,3 +56,4 @@ def prompt_and_delete_cruft(output_dir): .format(output_dir) ) return False + diff --git a/complexity/serve.py b/complexity/serve.py index be5c484..b87ebc1 100755 --- a/complexity/serve.py +++ b/complexity/serve.py @@ -21,7 +21,7 @@ import SocketServer as socketserver -def serve_static_site(output_dir, port=9090): +def serve_static_site(output_dir, address='127.0.0.1', port=9090): """ Serve a directory containing static HTML files, on a specified port. @@ -34,8 +34,8 @@ def serve_static_site(output_dir, port=9090): # of-errno-98-address-already-in-use socketserver.TCPServer.allow_reuse_address = True - httpd = socketserver.TCPServer(("", port), Handler) - print("serving at port", port) + httpd = socketserver.TCPServer((address, port), Handler) + print("serving ", address, port) try: httpd.serve_forever() diff --git a/setup.py b/setup.py index deb9613..f722e03 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,8 @@ requirements = [ 'jinja2>=2.4', 'binaryornot>=0.1.1', - 'PyYAML>=3.10' + 'PyYAML>=3.10', + 'watchdog>=0.8.3' ] test_requirements = []