diff --git a/Dockerfile b/Dockerfile index 8ff4b8b..3480d3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,18 @@ FROM jupyter/minimal-notebook -MAINTAINER Brendan Rius +MAINTAINER Xaver Klemenschits USER root +# Install vim and ssh +RUN apt-get update +RUN apt-get install -y vim openssh-client + WORKDIR /tmp COPY ./ jupyter_c_kernel/ -RUN pip install --no-cache-dir jupyter_c_kernel/ -RUN cd jupyter_c_kernel && install_c_kernel --user +RUN pip install --no-cache-dir -e jupyter_c_kernel/ > piplog.txt +RUN cd jupyter_c_kernel && install_c_kernel --user > installlog.txt WORKDIR /home/$NB_USER/ diff --git a/MANIFEST.in b/MANIFEST.in index ce43725..16c2e89 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include jupyter_c_kernel/resources/master.c +include jupyter_c_kernel/resources/stdio_wrap.h include README.md LICENSE.txt diff --git a/README.md b/README.md index 7d96ad1..e7a3d06 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,25 @@ -# Minimal C kernel for Jupyter +# C kernel for Jupyter + +This project was forked from [https://github.com/brendan-rius/jupyter-c-kernel](brendan-rius/jupyter-c-kernel) as that project seems to have been abandoned. (PR is pending) + +This project includes fixes to many issues reported in [https://github.com/brendan-rius/jupyter-c-kernel](brendan-rius/jupyter-c-kernel), as well as the following additional features: + +* Option for buffered output to mimic command line behaviour (useful for teaching, default is on) +* Command line input via `scanf` and `getchar` +* Support for `C89`/`ANSI C` (all newer versions were already supported and still are) + +Following limitations compared to command line execution exist: + +* Input is always buffered due to limitations of the jupyter interface +* When using `-ansi` or `-std=C89`, glibc still has to support at least `C99` for the interfacing with jupyter (this should not be an issue on an OS made after 2000) ## Use with Docker (recommended) - * `docker pull brendanrius/jupyter-c-kernel` - * `docker run -p 8888:8888 brendanrius/jupyter-c-kernel` - * Copy the given URL containing the token, and browse to it. For instance: - - ``` +* `docker pull xaverklemenschits/jupyter-c-kernel` +* `docker run -p 8888:8888 xaverklemenschits/jupyter-c-kernel` +* Copy the given URL containing the token, and browse to it. For instance: + + ```bash Copy/paste this URL into your browser when you connect for the first time, to login with a token: http://localhost:8888/?token=66750c80bd0788f6ba15760aadz53beb9a9fb4cf8ac15ce8 @@ -16,17 +29,22 @@ Works only on Linux and OS X. Windows is not supported yet. If you want to use this project on Windows, please use Docker. - - * Make sure you have the following requirements installed: +* Make sure you have the following requirements installed: * gcc * jupyter * python 3 * pip -### Step-by-step: - * `pip install jupyter-c-kernel` - * `install_c_kernel` - * `jupyter-notebook`. Enjoy! +### Step-by-step + +```bash +git clone https://github.com/XaverKlemenschits/jupyter-c-kernel.git +cd jupyter-c-kernel +pip install -e . # for system install: sudo install . +cd jupyter_c_kernel && install_c_kernel --user # for sys install: sudo install_c_kernel +# now you can start the notebook +jupyter notebook +``` ## Example of notebook @@ -47,9 +65,10 @@ change the code in real-time in Docker. For that, just run the docker box like that: ```bash -git clone https://github.com/brendan-rius/jupyter-c-kernel.git +git clone https://github.com/XaverKlemenschits/jupyter-c-kernel.git cd jupyter-c-kernel -docker run -v $(pwd):/jupyter/jupyter_c_kernel/ -p 8888:8888 brendanrius/jupyter-c-kernel +docker build -t myName/jupyter . +docker run -v $(pwd):/tmp/jupyter_c_kernel/ -p 8888:8888 myName/jupyter ``` This clones the source, run the kernel, and binds the current folder (the one diff --git a/example-notebook.ipynb b/example-notebook.ipynb index dcd2844..95b78b5 100644 --- a/example-notebook.ipynb +++ b/example-notebook.ipynb @@ -10,9 +10,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -32,6 +30,7 @@ "\n", "int main() {\n", " printf(\"Hello world\\n\");\n", + " return 0;\n", "}" ] }, @@ -52,9 +51,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -98,9 +95,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -142,14 +137,14 @@ "kernelspec": { "display_name": "C", "language": "c", - "name": "c_kernel" + "name": "c" }, "language_info": { - "file_extension": "c", + "file_extension": ".c", "mimetype": "text/plain", - "name": "c" + "name": "text/x-c++src" } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } diff --git a/jupyter_c_kernel/install_c_kernel b/jupyter_c_kernel/install_c_kernel index fcd009d..4b23026 100644 --- a/jupyter_c_kernel/install_c_kernel +++ b/jupyter_c_kernel/install_c_kernel @@ -29,7 +29,7 @@ def install_my_kernel_spec(user=True, prefix=None): # TODO: Copy resources once they're specified print('Installing IPython kernel spec') - KernelSpecManager().install_kernel_spec(td, 'c', user=user, replace=True, prefix=prefix) + KernelSpecManager().install_kernel_spec(td, 'c', user=user, prefix=prefix) def _is_root(): diff --git a/jupyter_c_kernel/kernel.py b/jupyter_c_kernel/kernel.py index 7ed7e71..42dc854 100644 --- a/jupyter_c_kernel/kernel.py +++ b/jupyter_c_kernel/kernel.py @@ -14,7 +14,9 @@ class RealTimeSubprocess(subprocess.Popen): A subprocess that allows to read its stdout and stderr in real time """ - def __init__(self, cmd, write_to_stdout, write_to_stderr): + inputRequest = "" + + def __init__(self, cmd, write_to_stdout, write_to_stderr, read_from_stdin): """ :param cmd: the command to execute :param write_to_stdout: a callable that will be called with chunks of data from stdout @@ -22,8 +24,9 @@ def __init__(self, cmd, write_to_stdout, write_to_stderr): """ self._write_to_stdout = write_to_stdout self._write_to_stderr = write_to_stderr + self._read_from_stdin = read_from_stdin - super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) + super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) self._stdout_queue = Queue() self._stdout_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stdout, self._stdout_queue)) @@ -58,12 +61,28 @@ def read_all_from_queue(queue): size -= 1 return res - stdout_contents = read_all_from_queue(self._stdout_queue) - if stdout_contents: - self._write_to_stdout(stdout_contents) stderr_contents = read_all_from_queue(self._stderr_queue) if stderr_contents: - self._write_to_stderr(stderr_contents) + self._write_to_stderr(stderr_contents.decode()) + + stdout_contents = read_all_from_queue(self._stdout_queue) + if stdout_contents: + contents = stdout_contents.decode() + # if there is input request, make output and then + # ask frontend for input + start = contents.find(self.__class__.inputRequest) + if(start >= 0): + contents = contents.replace(self.__class__.inputRequest, '') + if(len(contents) > 0): + self._write_to_stdout(contents) + readLine = "" + while(len(readLine) == 0): + readLine = self._read_from_stdin() + # need to add newline since it is not captured by frontend + readLine += "\n" + self.stdin.write(readLine.encode()) + else: + self._write_to_stdout(contents) class CKernel(Kernel): @@ -72,24 +91,41 @@ class CKernel(Kernel): language = 'c' language_version = 'C11' language_info = {'name': 'c', - 'mimetype': 'text/plain', + 'mimetype': 'text/x-csrc', 'file_extension': '.c'} banner = "C kernel.\n" \ "Uses gcc, compiles in C11, and creates source code files and executables in temporary folder.\n" + main_head = "#include \n" \ + "#include \n" \ + "int main(){\n" + + main_foot = "\nreturn 0;\n}" + def __init__(self, *args, **kwargs): super(CKernel, self).__init__(*args, **kwargs) + self._allow_stdin = True + self.readOnlyFileSystem = False + self.bufferedOutput = True + self.linkMaths = True # always link math library + self.wAll = True # show all warnings by default + self.wError = False # but keep comipiling for warnings + self.standard = "c11" # default standard if none is specified self.files = [] mastertemp = tempfile.mkstemp(suffix='.out') os.close(mastertemp[0]) self.master_path = mastertemp[1] - filepath = path.join(path.dirname(path.realpath(__file__)), 'resources', 'master.c') + self.resDir = path.join(path.dirname(path.realpath(__file__)), 'resources') + filepath = path.join(self.resDir, 'master.c') subprocess.call(['gcc', filepath, '-std=c11', '-rdynamic', '-ldl', '-o', self.master_path]) def cleanup_files(self): """Remove all the temporary files created by the kernel""" + # keep the list of files create in case there is an exception + # before they can be deleted as usual for file in self.files: - os.remove(file) + if(os.path.exists(file)): + os.remove(file) os.remove(self.master_path) def new_temp_file(self, **kwargs): @@ -107,13 +143,27 @@ def _write_to_stdout(self, contents): def _write_to_stderr(self, contents): self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': contents}) + def _read_from_stdin(self): + return self.raw_input() + def create_jupyter_subprocess(self, cmd): return RealTimeSubprocess(cmd, - lambda contents: self._write_to_stdout(contents.decode()), - lambda contents: self._write_to_stderr(contents.decode())) + self._write_to_stdout, + self._write_to_stderr, + self._read_from_stdin) def compile_with_gcc(self, source_filename, binary_filename, cflags=None, ldflags=None): - cflags = ['-std=c11', '-fPIC', '-shared', '-rdynamic'] + cflags + cflags = ['-pedantic', '-fPIC', '-shared', '-rdynamic'] + cflags + if self.linkMaths: + cflags = cflags + ['-lm'] + if self.wError: + cflags = cflags + ['-Werror'] + if self.wAll: + cflags = cflags + ['-Wall'] + if self.readOnlyFileSystem: + cflags = ['-DREAD_ONLY_FILE_SYSTEM'] + cflags + if self.bufferedOutput: + cflags = ['-DBUFFERED_OUTPUT'] + cflags args = ['gcc', source_filename] + cflags + ['-o', binary_filename] + ldflags return self.create_jupyter_subprocess(args) @@ -123,9 +173,16 @@ def _filter_magics(self, code): 'ldflags': [], 'args': []} + actualCode = '' + for line in code.splitlines(): if line.startswith('//%'): - key, value = line[3:].split(":", 2) + magicSplit = line[3:].split(":", 2) + if(len(magicSplit) < 2): + self._write_to_stderr("[C kernel] Magic line starting with '//%' is missing a semicolon, ignoring.") + continue + + key, value = magicSplit key = key.strip().lower() if key in ['ldflags', 'cflags']: @@ -136,12 +193,45 @@ def _filter_magics(self, code): for argument in re.findall(r'(?:[^\s,"]|"(?:\\.|[^"])*")+', value): magics['args'] += [argument.strip('"')] - return magics + # always add empty line, so line numbers don't change + actualCode += '\n' + + # keep lines which did not contain magics + else: + actualCode += line + '\n' + + # add default standard if cflags does not contain one + if not any(item.startswith('-std=') for item in magics["cflags"]): + magics["cflags"] += ["-std=" + self.standard] + + return magics, actualCode + + # check whether int main() is specified, if not add it around the code + # also add common magics like -lm + def _add_main(self, magics, code): + # remove comments + tmpCode = re.sub(r"//.*", "", code) + tmpCode = re.sub(r"/\*.*?\*/", "", tmpCode, flags=re.M|re.S) + + x = re.search(r"int\s+main\s*\(", tmpCode) + + if not x: + code = self.main_head + code + self.main_foot + magics['cflags'] += ['-lm'] + + return magics, code def do_execute(self, code, silent, store_history=True, - user_expressions=None, allow_stdin=False): + user_expressions=None, allow_stdin=True): + + magics, code = self._filter_magics(code) - magics = self._filter_magics(code) + magics, code = self._add_main(magics, code) + + # replace stdio with wrapped version + headerDir = "\"" + self.resDir + "/stdio_wrap.h" + "\"" + code = code.replace("", headerDir) + code = code.replace("\"stdio.h\"", headerDir) with self.new_temp_file(suffix='.c') as source_file: source_file.write(code) @@ -155,14 +245,28 @@ def do_execute(self, code, silent, store_history=True, self._write_to_stderr( "[C kernel] GCC exited with code {}, the executable will not be executed".format( p.returncode)) + + # delete source files before exit + os.remove(source_file.name) + os.remove(binary_file.name) + return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} p = self.create_jupyter_subprocess([self.master_path, binary_file.name] + magics['args']) while p.poll() is None: p.write_contents() + + # wait for threads to finish, so output is always shown + p._stdout_thread.join() + p._stderr_thread.join() + p.write_contents() + # now remove the files we have just created + os.remove(source_file.name) + os.remove(binary_file.name) + if p.returncode != 0: self._write_to_stderr("[C kernel] Executable exited with code {}".format(p.returncode)) return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} diff --git a/jupyter_c_kernel/resources/stdio_wrap.h b/jupyter_c_kernel/resources/stdio_wrap.h new file mode 100644 index 0000000..7b87d26 --- /dev/null +++ b/jupyter_c_kernel/resources/stdio_wrap.h @@ -0,0 +1,329 @@ +#ifndef STDIO_WRAP_H +#define STDIO_WRAP_H + +#include +#include +#include +#include +#include + +/* Figure out used C standard. + __STDC_VERSION__ is not always defined until C99. + If it is not defined, set standard to C89. + It is safest to set it by hand, to make sure */ +#ifdef __STDC_VERSION__ +#if __STDC_VERSION__ <= 199409L +#define C89_SUPPORT +#endif /* __STDC_VERSION__ <= 199409L */ +#else /* __STDC_VERSION__ */ +#define C89_SUPPORT +#endif /* __STDC_VERSION__ */ + +/* output functions to replicate terminal behaviour */ +#ifdef BUFFERED_OUTPUT +/* buffer for all output */ +/* TODO allocate this dynamically */ +static char outputBuff[1<<10] = ""; +static char attachedOutputFlush = 0; + +void flush_all_output() { + printf("%s", outputBuff); + fflush(stdout); + outputBuff[0] = '\0'; +} + +/* Flush all output on exit */ +void attachOutputFlush() { + if(attachedOutputFlush == 0){ + int error = atexit(flush_all_output); + if(error != 0) { + fprintf(stderr, "ERROR: Could not set exit function! Error %d\n", error); + } + attachedOutputFlush = 1; + } +} + +/* this function is called to check whether there + is a '\n' in the output that should be flushed */ +void flush_until_newline() { + long i = 0; + long length = strlen(outputBuff); + for(; i < length; ++i) { + if(outputBuff[i] == '\n') { + char *printBuff = malloc(i+2); + strncpy(printBuff, outputBuff, i+1); + printBuff[i+1] = '\0'; + printf("%s", printBuff); + free(printBuff); + /* now remove the printed string from the buffer + and start again */ + { + long a = 0; + ++i; + /* +1 to include \0 */ + for(; i < length + 1; ++a, ++i) { + outputBuff[a] = outputBuff[i]; + } + i = 0; + length = strlen(outputBuff); + } + } + } +} + +/* for printf, print all to a string. + Then cycle through all chars and see if \n is + written. If there is one, flush the output, otherwise + write to buffer */ +int printf_wrap(const char *format, ...) { + /* append output to buffer */ + va_list arglist; + int result; + va_start( arglist, format ); + result = vsprintf(outputBuff + strlen(outputBuff), format, arglist); + va_end( arglist ); + + /* Now flush if there is a reason to */ + flush_until_newline(); + + attachOutputFlush(); + + return result; +} + +int putchar_wrap(int c) { + long length = strlen(outputBuff); + outputBuff[length] = (char)c; + outputBuff[length+1] = '\0'; + if(c == '\n') { + flush_until_newline(); + } + + attachOutputFlush(); + + return c; +} + +int fflush_wrap(FILE* stream) { + if(stream == stdout) { + flush_all_output(); + } + return fflush(stream); +} + +int fclose_wrap(FILE* stream) { + if(stream == stdout) { + flush_all_output(); + } + return fclose(stream); +} +#endif /* BUFFERED_OUTPUT */ + +/* Need input buffer to know whether we need another input request */ +/* TODO allocate this dynamically */ +static char inputBuff[1<<10] = ""; +static long scanf_wrap_number_read = 0; + +/* read remaining input into buffer so it can be used in next call */ +void readIntoBuffer(void) { + long length = strlen(inputBuff); + char nextChar = 0; + while((nextChar = getchar()) != '\n' && nextChar != EOF){ + inputBuff[length++] = nextChar; + } + inputBuff[length++] = '\n'; + inputBuff[length] = '\0'; +} + +/* check whether input request is needed */ +char checkInputRequest(void) { + const long length = strlen(inputBuff); + long i = 0; + for(; i < length && isspace(inputBuff[i]); ++i); + return i == length; +} + +/* Define the input functions to overload the old ones */ +/* Wrapping of scanf depends on standard */ +#ifdef C89_SUPPORT +/* Need to define vscanf for c89. + TODO: This is a bit risky, since the underlying glibc does not + have to include this if it is old. If it does not, linking will fail. + The robust way would be readin via sscanf. */ + +/* Read formatted input from stdin into argument list ARG. + + This function is a possible cancellation point and therefore not + marked with __THROW. */ + extern int vsscanf (const char *__restrict __s, + const char *__restrict __format, _G_va_list __arg) + __THROW __attribute__ ((__format__ (__scanf__, 2, 0))); + + /* replace all % with %* to suppress read in and do test run */ + long find_scanf_length(const char *format) { + const long length = strlen(format); + /* allow for maximum of 50 format specifiers */ + char *formatString = malloc(length + 53); + long index = 0; + long formatIndex = 0; + for(; index < length; ++index, ++formatIndex) { + formatString[formatIndex] = format[index]; + if(format[index] == '%' && + (index + 1 < length && format[index + 1] != '%')) { + formatString[++formatIndex] = '*'; + } + } + /* add number readin */ + formatString[formatIndex++] = '%'; + formatString[formatIndex++] = 'n'; + formatString[formatIndex] = '\0'; + + /* now run and record how many characters were read */ + { + int readLength = 0; + sscanf(inputBuff, formatString, &readLength); + free(formatString); + + return readLength; + } + } +#endif /* C89_SUPPORT */ + +int scanf_wrap(const char *format, ...) { + char doRequest = checkInputRequest(); + char *formatString = 0; + + if(doRequest) { +#ifdef BUFFERED_OUTPUT + flush_all_output(); +#endif + printf(""); + fflush(stdout); + /* read everything from stdin into buffer */ + readIntoBuffer(); + } + + /* add %n to format string to get number of written chars */ + { + const long length = strlen(format); + formatString = malloc(length + 3); + strcpy(formatString, format); +#ifndef C89_SUPPORT + formatString[length] = '%'; + formatString[length + 1] = 'n'; + formatString[length + 2] = '\0'; +#else /* C89_SUPPORT */ + formatString[length] = '\0'; + /* In C89 we need to find how far scanf will read, by hand */ + scanf_wrap_number_read = find_scanf_length(format); +#endif /* C89_SUPPORT */ + } + + { + va_list arglist; + int result; + va_start(arglist, format); + result = vsscanf(inputBuff, formatString, arglist); + va_end(arglist); + + /* now move inputBuff up */ + { + const long length = strlen(inputBuff); + long index = scanf_wrap_number_read; + long a = 0; + /* +1 to include \0 */ + for(; index < length + 1; ++a, ++index) { + inputBuff[a] = inputBuff[index]; + } + } + + free(formatString); + return result; + } +} + +int getchar_wrap(void){ + /* check if there is still something in the input buffer*/ + char input = 0; + long length = strlen(inputBuff); + if(length <= 0) { +#ifdef BUFFERED_OUTPUT + flush_all_output(); +#endif + printf(""); + fflush(stdout); + + readIntoBuffer(); + } + + input = inputBuff[0]; + { + long i = 1; + long length = strlen(inputBuff) + 1; + /* shift all chars one to the left */ + for(; i < length; ++i){ + inputBuff[i-1] = inputBuff[i]; + if(inputBuff[i] == '\0') { + break; + } + } + } + + return input; +} + +/* Replace all the necessary input functions + depending on the language version used */ +#ifndef C89_SUPPORT +/* Need double hashes in case there are no __VA_ARGS__*/ +#define scanf(format, ...) scanf_wrap(format, ##__VA_ARGS__, &scanf_wrap_number_read) +#else /* C89_SUPPORT */ +/* Since there are no variadic macros in C89, this is the only way + although it is horrible */ +#define scanf scanf_wrap +#endif /* C89_SUPPORT */ + +#define getchar() getchar_wrap() + +/* Output defines */ +#ifdef BUFFERED_OUTPUT +#define printf printf_wrap +#define putchar putchar_wrap +#define fflush fflush_wrap +#define fclose fclose_wrap +#endif /* BUFFERED_OUTPUT */ + +/* Replace FILE write operations for read-only systems */ +#ifdef READ_ONLY_FILE_SYSTEM + +/* Define wrapping functions */ +/* Output that the fopen succeeded and return some valid pointer */ +FILE *fopen_wrap(const char *filename, const char *modes) { + static long stream = 0x1FFFF0000; +#ifdef SHOW_FILE_IO_VERBOSE + printf("\x01b[42m"); + printf("\"%s\" opened in mode \"%s\"\n", filename, modes); + printf("\x01b[0m"); +#endif /* SHOW_FILE_IO_VERBOSE */ + return (FILE*)stream++; +} + +int fprintf_wrap(FILE* stream, const char* format, ...) { + printf("\x01b[42m"); + printf("%p:", stream); + printf("\x01b[0m"); + va_list arglist; + va_start( arglist, format ); + int result = vprintf(format, arglist); + va_end( arglist ); + return result; +} + +/* Replace all the necessary input functions */ +#define fopen(file, mode) fopen_wrap(file, mode) + +#define fprintf(stream, format, ...) fprintf_wrap(stream, format, ##__VA_ARGS__) + +#endif /* READ_ONLY_FILE_SYSTEM */ + +#endif /* STDIO_WRAP_H */