diff --git a/.gitignore b/.gitignore index 4945b7a..091e4fc 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ src/html_entities.h *.o /fuzzing/bin /fuzzing/testing +venv* diff --git a/Dockerfile.wheel b/Dockerfile.wheel index 352bb08..abfdf4d 100644 --- a/Dockerfile.wheel +++ b/Dockerfile.wheel @@ -1,21 +1,70 @@ -FROM python:3.8-buster - +FROM python:2.7 ENV DEBIAN_FRONTEND noninteractive +ENV SRC_DIR /opt/snudown +ENV WHEEL_OUTPUT_DIR /tmp/dist +RUN mkdir -p $SRC_DIR $WHEEL_OUTPUT_DIR +WORKDIR $SRC_DIR +ADD scripts ./scripts +RUN scripts/ci-setup.sh +ADD . $SRC_DIR +ARG SNUDOWN_VERSION +RUN python setup.py install +RUN python test_snudown.py +CMD python -m pip wheel --wheel-dir=$WHEEL_OUTPUT_DIR . + -RUN apt-get update && \ - apt-get install -y \ - python \ - python-dev \ - python-pip \ - build-essential \ - gperf \ - software-properties-common +FROM python:3.9 +ENV DEBIAN_FRONTEND noninteractive +ENV SRC_DIR /opt/snudown +ENV WHEEL_OUTPUT_DIR /tmp/dist +RUN mkdir -p $SRC_DIR $WHEEL_OUTPUT_DIR +WORKDIR $SRC_DIR +ADD scripts ./scripts +RUN scripts/ci-setup.sh +ADD . $SRC_DIR +ARG SNUDOWN_VERSION +RUN python setup.py install +RUN python test_snudown.py +CMD python -m pip wheel --wheel-dir=$WHEEL_OUTPUT_DIR . +FROM python:3.10 +ENV DEBIAN_FRONTEND noninteractive ENV SRC_DIR /opt/snudown ENV WHEEL_OUTPUT_DIR /tmp/dist +RUN mkdir -p $SRC_DIR $WHEEL_OUTPUT_DIR +WORKDIR $SRC_DIR +ADD scripts ./scripts +RUN scripts/ci-setup.sh +ADD . $SRC_DIR +ARG SNUDOWN_VERSION +RUN python setup.py install +RUN python test_snudown.py +CMD python -m pip wheel --wheel-dir=$WHEEL_OUTPUT_DIR . +FROM python:3.11 +ENV DEBIAN_FRONTEND noninteractive +ENV SRC_DIR /opt/snudown +ENV WHEEL_OUTPUT_DIR /tmp/dist RUN mkdir -p $SRC_DIR $WHEEL_OUTPUT_DIR WORKDIR $SRC_DIR +ADD scripts ./scripts +RUN scripts/ci-setup.sh +ADD . $SRC_DIR +ARG SNUDOWN_VERSION +RUN python setup.py install +RUN python test_snudown.py +CMD python -m pip wheel --wheel-dir=$WHEEL_OUTPUT_DIR . +FROM python:3.12 +ENV DEBIAN_FRONTEND noninteractive +ENV SRC_DIR /opt/snudown +ENV WHEEL_OUTPUT_DIR /tmp/dist +RUN mkdir -p $SRC_DIR $WHEEL_OUTPUT_DIR +WORKDIR $SRC_DIR +ADD scripts ./scripts +RUN scripts/ci-setup.sh ADD . $SRC_DIR -CMD pip3 wheel --wheel-dir=$WHEEL_OUTPUT_DIR . ; pip2 wheel --wheel-dir=$WHEEL_OUTPUT_DIR . +ARG SNUDOWN_VERSION +RUN python setup.py install +RUN python test_snudown.py +CMD python -m pip wheel --wheel-dir=$WHEEL_OUTPUT_DIR . diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ba56abf --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2023 Reddit Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. diff --git a/Makefile b/Makefile index 2c1b7c1..510677c 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,27 @@ PACKAGE := snudown -VERSION := $(shell grep -oP 'SNUDOWN_VERSION "\K\d.\d.\d' snudown.c) +DRONE_TAG ?= 0.0.0 +SNUDOWN_VERSION := $(DRONE_TAG) +export SNUDOWN_VERSION DOCKERFILE := Dockerfile.wheel -default: clean build run +.PHONY: default build build-linux build-macos clean -build: - echo $(VERSION) - docker build \ - -t $(PACKAGE):$(VERSION) \ - -f $(DOCKERFILE) \ - . +default: clean build-linux build-macos -run: - mkdir -p dist - docker run \ - --rm \ - -v `pwd`/dist:/tmp/dist \ - -it \ - $(PACKAGE):$(VERSION) +build: build-linux build-macos + +build-linux: + bash scripts/build-linux.sh 2.7 + bash scripts/build-linux.sh 3.9 + bash scripts/build-linux.sh 3.10 + bash scripts/build-linux.sh 3.11 + bash scripts/build-linux.sh 3.12 + +build-macos: + bash scripts/build-macos.sh 3.9 + bash scripts/build-macos.sh 3.10 + bash scripts/build-macos.sh 3.11 + bash scripts/build-macos.sh 3.12 clean: rm -rf build diff --git a/README.markdown b/README.markdown index 2d43a23..dd9a38a 100644 --- a/README.markdown +++ b/README.markdown @@ -7,26 +7,13 @@ Markdown parser used by GitHub, with Python integration added. Setup for development on Mac OS X -------------------------------- -1. From `~/src/snudown` run `$ python setup.py build` +1. From `~/src/snudown` run `$ python setup.py develop` 2. If this is successful, there will now be a `snudown.so` file in the `/snudown/build/lib.< os info >-< python version number>` directory -3. From within the `/lib.< os info >-< python version number>` directory, start a python interpreter +3. Verify that the tests pass ``` - ->>> import snudown - ->>> print(snudown.__file__) -snudown.so - ->>> snudown.markdown('[hi](http://www.reddit.com)') -'

hi

\n' - ->>> quit() +$ python test_snudown.py ``` -4. Verify that the tests pass -``` -$ PYTHONPATH="$(pwd)" python ../../test_snudown.py -``` -5. Verify that all the previous steps work for both Python 2 AND Python 3 +4. Verify that all the previous steps work for both Python 2 AND Python 3 Install for general use @@ -34,10 +21,20 @@ Install for general use Run `setup.py install` to install the module. -For Mac OS X: -1. Install `afl-fuzz` via homebrew: `brew install afl-fuzz` -2. You can now install the module via `python setup.py install` -3. You may also compile snudown using the Makefile directly if you so wish +Building for production use +--------------------------- + +* Merge your approved PR into `master` +* Create a github version with the associated tag, ex. `v1.7.2` +* Let drone build and upload python linux packages to internal pypi registry + +Building for local dev use +--------------------------- + +For MacOS builds: +* Run `make build-macos` to build the MacOS wheels +* Upload the wheels (in `dist/`) to internal dev pypi registry +* Your MacOS build should now be available for local development. Thanks diff --git a/html/html.c b/html/html.c index 44b69a2..9f678fa 100755 --- a/html/html.c +++ b/html/html.c @@ -413,7 +413,8 @@ static void rndr_html_tag(struct buf *ob, const struct buf *text, void *opaque, char* tagname, char** whitelist, int tagtype) { - size_t i, x, z, in_str = 0, seen_equals = 0, done = 0, done_attr = 0, reset = 0; + size_t i, x, z, seen_equals = 0, done = 0, done_attr = 0, reset = 0; + uint8_t in_str = 0; struct buf *attr; struct buf *value; char c; @@ -471,7 +472,7 @@ rndr_html_tag(struct buf *ob, const struct buf *text, void *opaque, seen_equals = 1; break; default: - if(seen_equals && in_str || !seen_equals) { + if((seen_equals && in_str) || !seen_equals) { bufputc(seen_equals ? value : attr, c); } break; @@ -723,6 +724,7 @@ sdhtml_toc_renderer(struct sd_callbacks *callbacks, struct html_renderopt *optio NULL, NULL, NULL, + NULL, toc_header, NULL, NULL, diff --git a/scripts/build-linux.sh b/scripts/build-linux.sh new file mode 100755 index 0000000..9c22393 --- /dev/null +++ b/scripts/build-linux.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ex + +VERSION=${1} + +echo build ${VERSION} +docker build \ + --platform linux/amd64 \ + --target py${VERSION} \ + --build-arg SNUDOWN_VERSION \ + -t test:py${VERSION} \ + -f Dockerfile.wheel \ + . + +mkdir -p dist +# linux +docker run \ + --platform linux/amd64 \ + --rm \ + -e SNUDOWN_VERSION \ + -v `pwd`/dist:/tmp/dist \ + -it \ + test:py${VERSION} diff --git a/scripts/build-macos.sh b/scripts/build-macos.sh new file mode 100755 index 0000000..95ec76e --- /dev/null +++ b/scripts/build-macos.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ex + +#export CC=/opt/homebrew/bin/gcc-13 + +VERSION=${1} + +if [ ! -d "venv-${VERSION}" ]; then + python${VERSION} -m venv venv-${VERSION} +fi + +source venv-${VERSION}/bin/activate +rm -rf build +python -m pip install -U pip wheel setuptools +python setup.py install +python test_snudown.py +python -m pip wheel --wheel-dir=dist . +python test_snudown.py diff --git a/scripts/build-wheel.sh b/scripts/build-wheel.sh new file mode 100755 index 0000000..27a3b1d --- /dev/null +++ b/scripts/build-wheel.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +export WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:-dist} +python -m pip install -U pip wheel setuptools +python setup.py install +python test_snudown.py +python -m pip wheel --wheel-dir=${WHEEL_OUTPUT_DIR} . +ls -la dist + diff --git a/scripts/ci-setup.sh b/scripts/ci-setup.sh new file mode 100755 index 0000000..cc80675 --- /dev/null +++ b/scripts/ci-setup.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +apt-get update && apt-get install -y \ + python \ + python-dev \ + build-essential \ + gperf \ + software-properties-common diff --git a/setup.py b/setup.py index a780cb0..12d5d72 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,17 @@ -from distutils.spawn import find_executable from setuptools import setup, Extension from setuptools.command.build_ext import build_ext +import sysconfig + +try: + from distutils.spawn import find_executable +except ImportError: + from shutil import which as find_executable -import re import os import subprocess import fnmatch + def c_files_in(directory): paths = [] names = os.listdir(directory) @@ -20,14 +25,15 @@ def process_gperf_file(gperf_file, output_file): raise Exception("Couldn't find `gperf`, is it installed?") subprocess.check_call(["gperf", gperf_file, "--output-file=%s" % output_file]) -version = None -version_re = re.compile(r'^#define\s+SNUDOWN_VERSION\s+"([^"]+)"$') -with open('snudown.c', 'r') as f: - for line in f: - m = version_re.match(line) - if m: - version = m.group(1) -assert version + +# version is defined in environment +version = os.environ.get("SNUDOWN_VERSION", "0.0.0") +if version.startswith("v"): + version = version[1:] + +# make sure the version string is escaped properly +extra_compile_args = ["-DSNUDOWN_VERSION=\"\\\"%s\\\"\"" % version] +extra_compile_args.extend(sysconfig.get_config_var('CFLAGS').split()) class GPerfingBuildExt(build_ext): @@ -35,6 +41,7 @@ def run(self): process_gperf_file("src/html_entities.gperf", "src/html_entities.h") build_ext.run(self) + setup( name='snudown', version=version, @@ -42,12 +49,13 @@ def run(self): author_email='vicent@github.com', license='MIT', test_suite="test_snudown.test_snudown", - cmdclass={'build_ext': GPerfingBuildExt,}, + cmdclass={'build_ext': GPerfingBuildExt}, ext_modules=[ Extension( name='snudown', sources=['snudown.c'] + c_files_in('src/') + c_files_in('html/'), - include_dirs=['src', 'html'] + include_dirs=['src', 'html'], + extra_compile_args=extra_compile_args, ) ], ) diff --git a/snudown.c b/snudown.c index dd891b9..e415ee7 100644 --- a/snudown.c +++ b/snudown.c @@ -5,8 +5,6 @@ #include "html.h" #include "autolink.h" -#define SNUDOWN_VERSION "1.7.0" - enum snudown_renderer_mode { RENDERER_USERTEXT = 0, RENDERER_WIKI, diff --git a/src/autolink.c b/src/autolink.c index 8d0e39a..8623f5e 100644 --- a/src/autolink.c +++ b/src/autolink.c @@ -388,7 +388,7 @@ sd_autolink__subreddit( do { size_t start = link_end; - int max_length = 24; + size_t max_length = 24; /* special case: /r/reddit.com (only subreddit containing '.'). */ if ( size >= link_end+10 && strncasecmp((char*)data+link_end, "reddit.com", 10) == 0 ) { diff --git a/src/html_blocks.h b/src/html_blocks.h index f90ee70..2d0e0ae 100644 --- a/src/html_blocks.h +++ b/src/html_blocks.h @@ -87,9 +87,7 @@ inline #endif #endif static unsigned int -hash_block_tag (str, len) - register const char *str; - register unsigned int len; +hash_block_tag (register const char *str, register unsigned int len) { static const unsigned char asso_values[] = { @@ -141,9 +139,7 @@ __attribute__ ((__gnu_inline__)) #endif #endif const char * -find_block_tag (str, len) - register const char *str; - register unsigned int len; +find_block_tag (register const char *str, register unsigned int len) { enum { diff --git a/src/html_entities.gperf b/src/html_entities.gperf index f94e3c9..8be8e21 100644 --- a/src/html_entities.gperf +++ b/src/html_entities.gperf @@ -14,7 +14,7 @@ const u_int32_t MAX_NUM_ENTITY_VAL = 0x10ffff; * used to avoid dealing with overflows. */ const size_t MAX_NUM_ENTITY_LEN = 7; -inline int is_valid_numeric_entity(uint32_t entity_val) +int is_valid_numeric_entity(uint32_t entity_val) { /* Some XML parsers will choke on entities with certain * values (mostly control characters.) diff --git a/src/markdown.c b/src/markdown.c index d49725c..f69cb03 100644 --- a/src/markdown.c +++ b/src/markdown.c @@ -26,12 +26,32 @@ #include #include -#if __GLIBC__ >= 2 && __GLIBC_MINOR >= 25 + #include + +#if defined __GLIBC_PREREQ && !defined __UCLIBC__ +#define GLIBC_PREREQ(M, m) (__GLIBC_PREREQ(M, m)) +#else +#define GLIBC_PREREQ(M, m) 0 +#endif + +#ifndef HAVE_GETRANDOM +#define HAVE_GETRANDOM (GLIBC_PREREQ(2,25) && __linux__) +#endif + +/* + * some systems define `getentropy` instead of `getrandom` + */ +ssize_t markdown_getrandom(void *buf, size_t buflen, unsigned int flags) +{ +#if HAVE_GETRANDOM + return getrandom(buf, buflen, flags); #else -# define getrandom backport_getrandom -# include + // arc4random_buf is always successful + arc4random_buf(buf, buflen); + return buflen; #endif +} #if defined(_WIN32) #define strncasecmp _strnicmp @@ -139,11 +159,6 @@ uint8_t sip_hash_key[SIP_HASH_KEY_LEN]; * HELPER FUNCTIONS * ***************************/ -int backport_getrandom(void *buf, size_t buflen, unsigned int flags) -{ - return (int)syscall(SYS_getrandom, buf, buflen, flags); -} - static inline struct buf * rndr_newbuf(struct sd_markdown *rndr, int type) { @@ -2282,7 +2297,7 @@ parse_table_header( size_t *columns, int **column_data) { - int pipes; + size_t pipes; size_t i = 0, col, header_end, under_end; pipes = 0; @@ -2604,6 +2619,9 @@ is_ref(const uint8_t *data, size_t beg, size_t end, size_t *last, struct link_re ref->title = bufnew(title_end - title_offset); bufput(ref->title, data + title_offset, title_end - title_offset); } + else { + ref->title = 0; + } } return 1; @@ -2655,7 +2673,7 @@ sd_markdown_new( return NULL; if (!sip_hash_key_init) { - if (getrandom(sip_hash_key, SIP_HASH_KEY_LEN, 0) < SIP_HASH_KEY_LEN) + if (markdown_getrandom(sip_hash_key, SIP_HASH_KEY_LEN, 0) < SIP_HASH_KEY_LEN) return NULL; sip_hash_key_init = 1; } diff --git a/test_snudown.py b/test_snudown.py index d1b8df4..22db5e8 100644 --- a/test_snudown.py +++ b/test_snudown.py @@ -518,6 +518,7 @@ '

\n', } + class SnudownTestCase(unittest.TestCase): def __init__(self, renderer=snudown.RENDERER_USERTEXT): self.renderer = renderer @@ -555,6 +556,7 @@ def test_snudown(): return suite + if __name__ == '__main__': runner = unittest.TextTestRunner() runner.run(test_snudown())