From 42defa224900ad2e4aa8007419c729eb8ea78fe5 Mon Sep 17 00:00:00 2001 From: nleanba Date: Thu, 1 Sep 2022 19:12:21 +0200 Subject: [PATCH 01/35] move old stuff out of the way --- .gitignore | 65 -- .vscode/settings.json | 4 +- 3doc.config.js | 0 LICENSE | 2 +- find-draft.txt | 2 - .dockerignore => old/.dockerignore | 0 old/.gitignore | 9 + DEV-README.md => old/DEV-README.md | 0 Dockerfile => old/Dockerfile | 0 config-tdb.ttl => old/config-tdb.ttl | 0 .../dev-docker-compose.yml | 0 docker-cmd.sh => old/docker-cmd.sh | 0 docker-compose.yml => old/docker-compose.yml | 0 {lib => old/lib}/datastore.js | 0 {lib => old/lib}/metadeleter.js | 0 {lib => old/lib}/metafinder.js | 0 {lib => old/lib}/metastorer.js | 0 {lib => old/lib}/pdfprocessor.js | 0 {lib => old/lib}/server.js | 0 package.json => old/package.json | 0 tdt.fish => old/tdt.fish | 0 yarn.lock | 624 ------------------ 22 files changed, 13 insertions(+), 693 deletions(-) delete mode 100644 .gitignore delete mode 100644 3doc.config.js delete mode 100644 find-draft.txt rename .dockerignore => old/.dockerignore (100%) create mode 100644 old/.gitignore rename DEV-README.md => old/DEV-README.md (100%) rename Dockerfile => old/Dockerfile (100%) rename config-tdb.ttl => old/config-tdb.ttl (100%) rename dev-docker-compose.yml => old/dev-docker-compose.yml (100%) rename docker-cmd.sh => old/docker-cmd.sh (100%) rename docker-compose.yml => old/docker-compose.yml (100%) rename {lib => old/lib}/datastore.js (100%) rename {lib => old/lib}/metadeleter.js (100%) rename {lib => old/lib}/metafinder.js (100%) rename {lib => old/lib}/metastorer.js (100%) rename {lib => old/lib}/pdfprocessor.js (100%) rename {lib => old/lib}/server.js (100%) rename package.json => old/package.json (100%) rename tdt.fish => old/tdt.fish (100%) delete mode 100644 yarn.lock diff --git a/.gitignore b/.gitignore deleted file mode 100644 index c797f99..0000000 --- a/.gitignore +++ /dev/null @@ -1,65 +0,0 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# TypeScript v1 declaration files -typings/ - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variables file -.env - -# next.js build output -.next - -blobs - -fuseki-base \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 3d41712..85ece8f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,5 @@ { - "npm.packageManager": "yarn" + "npm.packageManager": "yarn", + "deno.enable": true, + "deno.unstable": true } \ No newline at end of file diff --git a/3doc.config.js b/3doc.config.js deleted file mode 100644 index e69de29..0000000 diff --git a/LICENSE b/LICENSE index e2b8549..2ec5ef9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 Reto Gmür +Copyright (c) 2022 Noam Bachmann & Reto Gmür Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/find-draft.txt b/find-draft.txt deleted file mode 100644 index df2b38d..0000000 --- a/find-draft.txt +++ /dev/null @@ -1,2 +0,0 @@ -Encode where necessary: GET /doc?tag=pay&tag=prority(>3)&tag=not(personal)&search=helsinki - /doc?tag=pay&tag=prority(>3)¬tag=personal&text=helsinki diff --git a/.dockerignore b/old/.dockerignore similarity index 100% rename from .dockerignore rename to old/.dockerignore diff --git a/old/.gitignore b/old/.gitignore new file mode 100644 index 0000000..ccf4377 --- /dev/null +++ b/old/.gitignore @@ -0,0 +1,9 @@ +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +node_modules +blobs +fuseki-base \ No newline at end of file diff --git a/DEV-README.md b/old/DEV-README.md similarity index 100% rename from DEV-README.md rename to old/DEV-README.md diff --git a/Dockerfile b/old/Dockerfile similarity index 100% rename from Dockerfile rename to old/Dockerfile diff --git a/config-tdb.ttl b/old/config-tdb.ttl similarity index 100% rename from config-tdb.ttl rename to old/config-tdb.ttl diff --git a/dev-docker-compose.yml b/old/dev-docker-compose.yml similarity index 100% rename from dev-docker-compose.yml rename to old/dev-docker-compose.yml diff --git a/docker-cmd.sh b/old/docker-cmd.sh similarity index 100% rename from docker-cmd.sh rename to old/docker-cmd.sh diff --git a/docker-compose.yml b/old/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to old/docker-compose.yml diff --git a/lib/datastore.js b/old/lib/datastore.js similarity index 100% rename from lib/datastore.js rename to old/lib/datastore.js diff --git a/lib/metadeleter.js b/old/lib/metadeleter.js similarity index 100% rename from lib/metadeleter.js rename to old/lib/metadeleter.js diff --git a/lib/metafinder.js b/old/lib/metafinder.js similarity index 100% rename from lib/metafinder.js rename to old/lib/metafinder.js diff --git a/lib/metastorer.js b/old/lib/metastorer.js similarity index 100% rename from lib/metastorer.js rename to old/lib/metastorer.js diff --git a/lib/pdfprocessor.js b/old/lib/pdfprocessor.js similarity index 100% rename from lib/pdfprocessor.js rename to old/lib/pdfprocessor.js diff --git a/lib/server.js b/old/lib/server.js similarity index 100% rename from lib/server.js rename to old/lib/server.js diff --git a/package.json b/old/package.json similarity index 100% rename from package.json rename to old/package.json diff --git a/tdt.fish b/old/tdt.fish similarity index 100% rename from tdt.fish rename to old/tdt.fish diff --git a/yarn.lock b/yarn.lock deleted file mode 100644 index 1ad8e52..0000000 --- a/yarn.lock +++ /dev/null @@ -1,624 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -accept@3.x.x: - version "3.0.2" - resolved "https://registry.yarnpkg.com/accept/-/accept-3.0.2.tgz#83e41cec7e1149f3fd474880423873db6c6cc9ac" - dependencies: - boom "7.x.x" - hoek "5.x.x" - -adm-zip@^0.4.16: - version "0.4.16" - resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.4.16.tgz#cf4c508fdffab02c269cbc7f471a875f05570365" - integrity sha512-TFi4HBKSGfIKsK5YCkKaaFG2m4PEDyViZmEwof3MTIgzimHLto6muaHVpbrljdIvIrFZzEq/p4nafOeLcYegrg== - -ajv-keywords@^3.1.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.2.0.tgz#e86b819c602cf8821ad637413698f1dec021847a" - -ajv@^6.1.0: - version "6.5.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.5.2.tgz#678495f9b82f7cca6be248dd92f59bff5e1f4360" - dependencies: - fast-deep-equal "^2.0.1" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.4.1" - uri-js "^4.2.1" - -ammo@3.x.x: - version "3.0.1" - resolved "https://registry.yarnpkg.com/ammo/-/ammo-3.0.1.tgz#c79ceeac36fb4e55085ea3fe0c2f42bfa5f7c914" - dependencies: - hoek "5.x.x" - -archiver-utils@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-2.1.0.tgz#e8a460e94b693c3e3da182a098ca6285ba9249e2" - integrity sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw== - dependencies: - glob "^7.1.4" - graceful-fs "^4.2.0" - lazystream "^1.0.0" - lodash.defaults "^4.2.0" - lodash.difference "^4.5.0" - lodash.flatten "^4.4.0" - lodash.isplainobject "^4.0.6" - lodash.union "^4.6.0" - normalize-path "^3.0.0" - readable-stream "^2.0.0" - -archiver@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/archiver/-/archiver-3.1.1.tgz#9db7819d4daf60aec10fe86b16cb9258ced66ea0" - integrity sha512-5Hxxcig7gw5Jod/8Gq0OneVgLYET+oNHcxgWItq4TbhOzRLKNAFUb9edAftiMKXvXfCB0vbGrJdZDNq0dWMsxg== - dependencies: - archiver-utils "^2.1.0" - async "^2.6.3" - buffer-crc32 "^0.2.1" - glob "^7.1.4" - readable-stream "^3.4.0" - tar-stream "^2.1.0" - zip-stream "^2.1.2" - -async@^2.6.3: - version "2.6.3" - resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" - integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg== - dependencies: - lodash "^4.17.14" - -b64@4.x.x: - version "4.0.0" - resolved "https://registry.yarnpkg.com/b64/-/b64-4.0.0.tgz#c37f587f0a383c7019e821120e8c3f58f0d22772" - -balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= - -base64-js@^1.0.2: - version "1.3.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" - integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== - -big-time@2.x.x: - version "2.0.1" - resolved "https://registry.yarnpkg.com/big-time/-/big-time-2.0.1.tgz#68c7df8dc30f97e953f25a67a76ac9713c16c9de" - -big.js@^3.1.3: - version "3.2.0" - resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e" - -bl@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.0.tgz#3611ec00579fd18561754360b21e9f784500ff88" - integrity sha512-EUAyP5UHU5hxF8BPT0LKW8gjYLhq1DQIcneOX/pL/m2Alo+OYDQAJlHq+yseMP50Os2nHXOSic6Ss3vSQeyf4A== - dependencies: - readable-stream "^3.0.1" - -boom@7.x.x: - version "7.2.0" - resolved "https://registry.yarnpkg.com/boom/-/boom-7.2.0.tgz#2bff24a55565767fde869ec808317eb10c48e966" - dependencies: - hoek "5.x.x" - -bounce@1.x.x: - version "1.2.0" - resolved "https://registry.yarnpkg.com/bounce/-/bounce-1.2.0.tgz#e3bac68c73fd256e38096551efc09f504873c8c8" - dependencies: - boom "7.x.x" - hoek "5.x.x" - -brace-expansion@^1.1.7: - version "1.1.11" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" - integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== - dependencies: - balanced-match "^1.0.0" - concat-map "0.0.1" - -buffer-crc32@^0.2.1, buffer-crc32@^0.2.13: - version "0.2.13" - resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" - integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= - -buffer@^5.1.0: - version "5.4.3" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.4.3.tgz#3fbc9c69eb713d323e3fc1a895eee0710c072115" - integrity sha512-zvj65TkFeIt3i6aj5bIvJDzjjQQGs4o/sNoezg1F1kYap9Nu2jcUdpwzRSJTHMMzG0H7bZkn4rNQpImhuxWX2A== - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - -call@5.x.x: - version "5.0.1" - resolved "https://registry.yarnpkg.com/call/-/call-5.0.1.tgz#ac1b5c106d9edc2a17af2a4a4f74dd4f0c06e910" - dependencies: - boom "7.x.x" - hoek "5.x.x" - -catbox-memory@3.x.x: - version "3.1.2" - resolved "https://registry.yarnpkg.com/catbox-memory/-/catbox-memory-3.1.2.tgz#4aeec1bc994419c0f7e60087f172aaedd9b4911c" - dependencies: - big-time "2.x.x" - boom "7.x.x" - hoek "5.x.x" - -catbox@10.x.x: - version "10.0.2" - resolved "https://registry.yarnpkg.com/catbox/-/catbox-10.0.2.tgz#e6ac1f35102d1a9bd07915b82e508d12b50a8bfa" - dependencies: - boom "7.x.x" - bounce "1.x.x" - hoek "5.x.x" - joi "13.x.x" - -compress-commons@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-2.1.1.tgz#9410d9a534cf8435e3fbbb7c6ce48de2dc2f0610" - integrity sha512-eVw6n7CnEMFzc3duyFVrQEuY1BlHR3rYsSztyG32ibGMW722i3C6IizEGMFmfMU+A+fALvBIwxN3czffTcdA+Q== - dependencies: - buffer-crc32 "^0.2.13" - crc32-stream "^3.0.1" - normalize-path "^3.0.0" - readable-stream "^2.3.6" - -concat-map@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" - integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= - -content@4.x.x: - version "4.0.5" - resolved "https://registry.yarnpkg.com/content/-/content-4.0.5.tgz#bc547deabc889ab69bce17faf3585c29f4c41bf2" - dependencies: - boom "7.x.x" - -core-util-is@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= - -crc32-stream@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-3.0.1.tgz#cae6eeed003b0e44d739d279de5ae63b171b4e85" - integrity sha512-mctvpXlbzsvK+6z8kJwSJ5crm7yBwrQMTybJzMw1O4lLGJqjlDCXY2Zw7KheiA6XBEcBmfLx1D88mjRGVJtY9w== - dependencies: - crc "^3.4.4" - readable-stream "^3.4.0" - -crc@^3.4.4: - version "3.8.0" - resolved "https://registry.yarnpkg.com/crc/-/crc-3.8.0.tgz#ad60269c2c856f8c299e2c4cc0de4556914056c6" - integrity sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ== - dependencies: - buffer "^5.1.0" - -cryptiles@4.x.x: - version "4.1.2" - resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-4.1.2.tgz#363c9ab5c859da9d2d6fb901b64d980966181184" - dependencies: - boom "7.x.x" - -emojis-list@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389" - -end-of-stream@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" - integrity sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q== - dependencies: - once "^1.4.0" - -fast-deep-equal@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49" - -fast-json-stable-stringify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" - -fs-constants@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" - integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== - -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= - -glob@^7.1.4: - version "7.1.4" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.4.tgz#aa608a2f6c577ad357e1ae5a5c26d9a8d1969255" - integrity sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - -graceful-fs@^4.2.0: - version "4.2.2" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.2.tgz#6f0952605d0140c1cfdb138ed005775b92d67b02" - integrity sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q== - -hapi-auth-basic@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/hapi-auth-basic/-/hapi-auth-basic-5.0.0.tgz#0438b00225e4f7baccd7f29e04b4fc5037c012b0" - integrity sha1-BDiwAiXk97rM1/KeBLT8UDfAErA= - dependencies: - boom "7.x.x" - hoek "5.x.x" - -hapi@^17.5.2: - version "17.5.2" - resolved "https://registry.yarnpkg.com/hapi/-/hapi-17.5.2.tgz#9c5823cdcdd17e5621ebc8928aefb144d033caac" - dependencies: - accept "3.x.x" - ammo "3.x.x" - boom "7.x.x" - bounce "1.x.x" - call "5.x.x" - catbox "10.x.x" - catbox-memory "3.x.x" - heavy "6.x.x" - hoek "5.x.x" - joi "13.x.x" - mimos "4.x.x" - podium "3.x.x" - shot "4.x.x" - statehood "6.x.x" - subtext "6.x.x" - teamwork "3.x.x" - topo "3.x.x" - -heavy@6.x.x: - version "6.1.0" - resolved "https://registry.yarnpkg.com/heavy/-/heavy-6.1.0.tgz#1bbfa43dc61dd4b543ede3ff87db8306b7967274" - dependencies: - boom "7.x.x" - hoek "5.x.x" - joi "13.x.x" - -hoek@5.x.x: - version "5.0.3" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-5.0.3.tgz#b71d40d943d0a95da01956b547f83c4a5b4a34ac" - -ieee754@^1.1.4: - version "1.1.13" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" - integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== - -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2, inherits@^2.0.3, inherits@~2.0.3: - version "2.0.4" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" - integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== - -iron@5.x.x: - version "5.0.4" - resolved "https://registry.yarnpkg.com/iron/-/iron-5.0.4.tgz#003ed822f656f07c2b62762815f5de3947326867" - dependencies: - boom "7.x.x" - cryptiles "4.x.x" - hoek "5.x.x" - -isarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= - -isemail@3.x.x: - version "3.1.3" - resolved "https://registry.yarnpkg.com/isemail/-/isemail-3.1.3.tgz#64f37fc113579ea12523165c3ebe3a71a56ce571" - dependencies: - punycode "2.x.x" - -joi@13.x.x: - version "13.4.0" - resolved "https://registry.yarnpkg.com/joi/-/joi-13.4.0.tgz#afc359ee3d8bc5f9b9ba6cdc31b46d44af14cecc" - dependencies: - hoek "5.x.x" - isemail "3.x.x" - topo "3.x.x" - -json-schema-traverse@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" - -json5@^0.5.0: - version "0.5.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821" - -lazystream@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.0.tgz#f6995fe0f820392f61396be89462407bb77168e4" - integrity sha1-9plf4PggOS9hOWvolGJAe7dxaOQ= - dependencies: - readable-stream "^2.0.5" - -loader-utils@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.1.0.tgz#c98aef488bcceda2ffb5e2de646d6a754429f5cd" - dependencies: - big.js "^3.1.3" - emojis-list "^2.0.0" - json5 "^0.5.0" - -lodash.defaults@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" - integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw= - -lodash.difference@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c" - integrity sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw= - -lodash.flatten@^4.4.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f" - integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8= - -lodash.isplainobject@^4.0.6: - version "4.0.6" - resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" - integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs= - -lodash.union@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88" - integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg= - -lodash@^4.17.14: - version "4.17.15" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" - integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== - -mime-db@1.x.x: - version "1.35.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.35.0.tgz#0569d657466491283709663ad379a99b90d9ab47" - -mimos@4.x.x: - version "4.0.0" - resolved "https://registry.yarnpkg.com/mimos/-/mimos-4.0.0.tgz#76e3d27128431cb6482fd15b20475719ad626a5a" - dependencies: - hoek "5.x.x" - mime-db "1.x.x" - -minimatch@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== - dependencies: - brace-expansion "^1.1.7" - -nanoid@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-1.1.0.tgz#b18e806e1cdbfdbe030374d5cf08a48cbc80b474" - -nigel@3.x.x: - version "3.0.1" - resolved "https://registry.yarnpkg.com/nigel/-/nigel-3.0.1.tgz#48a08859d65177312f1c25af7252c1e07bb07c2a" - dependencies: - hoek "5.x.x" - vise "3.x.x" - -node-ensure@^0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7" - -node-fetch@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5" - -normalize-path@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" - integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== - -once@^1.3.0, once@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= - dependencies: - wrappy "1" - -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= - -pdfjs-dist@^2.0.489: - version "2.0.489" - resolved "https://registry.yarnpkg.com/pdfjs-dist/-/pdfjs-dist-2.0.489.tgz#63e54b292a86790a454697eb44d4347b8fbfad27" - dependencies: - node-ensure "^0.0.0" - worker-loader "^1.1.1" - -pez@4.x.x: - version "4.0.2" - resolved "https://registry.yarnpkg.com/pez/-/pez-4.0.2.tgz#0a7c81b64968e90b0e9562b398f390939e9c4b53" - dependencies: - b64 "4.x.x" - boom "7.x.x" - content "4.x.x" - hoek "5.x.x" - nigel "3.x.x" - -podium@3.x.x: - version "3.1.2" - resolved "https://registry.yarnpkg.com/podium/-/podium-3.1.2.tgz#b701429739cf6bdde6b3015ae6b48d400817ce9e" - dependencies: - hoek "5.x.x" - joi "13.x.x" - -process-nextick-args@~2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" - integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== - -punycode@2.x.x, punycode@^2.1.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" - -readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.3.6: - version "2.3.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" - integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" - -readable-stream@^3.0.1, readable-stream@^3.1.1, readable-stream@^3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc" - integrity sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - -safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - -safe-buffer@~5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" - integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== - -schema-utils@^0.4.0: - version "0.4.5" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.5.tgz#21836f0608aac17b78f9e3e24daff14a5ca13a3e" - dependencies: - ajv "^6.1.0" - ajv-keywords "^3.1.0" - -shot@4.x.x: - version "4.0.5" - resolved "https://registry.yarnpkg.com/shot/-/shot-4.0.5.tgz#c7e7455d11d60f6b6cd3c43e15a3b431c17e5566" - dependencies: - hoek "5.x.x" - joi "13.x.x" - -statehood@6.x.x: - version "6.0.6" - resolved "https://registry.yarnpkg.com/statehood/-/statehood-6.0.6.tgz#0dbd7c50774d3f61a24e42b0673093bbc81fa5f0" - dependencies: - boom "7.x.x" - bounce "1.x.x" - cryptiles "4.x.x" - hoek "5.x.x" - iron "5.x.x" - joi "13.x.x" - -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== - dependencies: - safe-buffer "~5.1.0" - -subtext@6.x.x: - version "6.0.7" - resolved "https://registry.yarnpkg.com/subtext/-/subtext-6.0.7.tgz#8e40a67901a734d598142665c90e398369b885f9" - dependencies: - boom "7.x.x" - content "4.x.x" - hoek "5.x.x" - pez "4.x.x" - wreck "14.x.x" - -tar-stream@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.1.0.tgz#d1aaa3661f05b38b5acc9b7020efdca5179a2cc3" - integrity sha512-+DAn4Nb4+gz6WZigRzKEZl1QuJVOLtAwwF+WUxy1fJ6X63CaGaUAxJRD2KEn1OMfcbCjySTYpNC6WmfQoIEOdw== - dependencies: - bl "^3.0.0" - end-of-stream "^1.4.1" - fs-constants "^1.0.0" - inherits "^2.0.3" - readable-stream "^3.1.1" - -teamwork@3.x.x: - version "3.0.1" - resolved "https://registry.yarnpkg.com/teamwork/-/teamwork-3.0.1.tgz#ff38c7161f41f8070b7813716eb6154036ece196" - -topo@3.x.x: - version "3.0.0" - resolved "https://registry.yarnpkg.com/topo/-/topo-3.0.0.tgz#37e48c330efeac784538e0acd3e62ca5e231fe7a" - dependencies: - hoek "5.x.x" - -uri-js@^4.2.1: - version "4.2.2" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" - dependencies: - punycode "^2.1.0" - -util-deprecate@^1.0.1, util-deprecate@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= - -vise@3.x.x: - version "3.0.0" - resolved "https://registry.yarnpkg.com/vise/-/vise-3.0.0.tgz#76ad14ab31669c50fbb0817bc0e72fedcbb3bf4c" - dependencies: - hoek "5.x.x" - -worker-loader@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-1.1.1.tgz#920d74ddac6816fc635392653ed8b4af1929fd92" - dependencies: - loader-utils "^1.0.0" - schema-utils "^0.4.0" - -wrappy@1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= - -wreck@14.x.x: - version "14.0.2" - resolved "https://registry.yarnpkg.com/wreck/-/wreck-14.0.2.tgz#89c17a9061c745ed1c3aebcb66ea181dbaab454c" - dependencies: - boom "7.x.x" - hoek "5.x.x" - -zip-stream@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-2.1.2.tgz#841efd23214b602ff49c497cba1a85d8b5fbc39c" - integrity sha512-ykebHGa2+uzth/R4HZLkZh3XFJzivhVsjJt8bN3GvBzLaqqrUdRacu+c4QtnUgjkkQfsOuNE1JgLKMCPNmkKgg== - dependencies: - archiver-utils "^2.1.0" - compress-commons "^2.1.1" - readable-stream "^3.4.0" From 3c7dcd8a7156531ee4120ed74075801dd48b6af7 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 15:14:13 +0200 Subject: [PATCH 02/35] defined all routes --- deno.jsonc | 11 +++++ src/deps.ts | 2 + src/handlers/count.ts | 3 ++ src/handlers/notImplemented.ts | 3 ++ src/helpers/processParams.ts | 88 ++++++++++++++++++++++++++++++++++ src/main.ts | 12 +++++ src/meta/finder.ts | 20 ++++++++ src/server/routes.ts | 85 ++++++++++++++++++++++++++++++++ src/server/server.ts | 52 ++++++++++++++++++++ 9 files changed, 276 insertions(+) create mode 100644 deno.jsonc create mode 100644 src/deps.ts create mode 100644 src/handlers/count.ts create mode 100644 src/handlers/notImplemented.ts create mode 100644 src/helpers/processParams.ts create mode 100644 src/main.ts create mode 100644 src/meta/finder.ts create mode 100644 src/server/routes.ts create mode 100644 src/server/server.ts diff --git a/deno.jsonc b/deno.jsonc new file mode 100644 index 0000000..86149a3 --- /dev/null +++ b/deno.jsonc @@ -0,0 +1,11 @@ +{ + "fmt": { + "files": { + "include": ["src/"] + } + }, + "tasks": { + "run": "deno run --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts", + "run-watch": "deno run --watch --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts" + } +} diff --git a/src/deps.ts b/src/deps.ts new file mode 100644 index 0000000..1ad18b0 --- /dev/null +++ b/src/deps.ts @@ -0,0 +1,2 @@ +export { serve } from "https://deno.land/std@0.153.0/http/mod.ts"; +export { encode } from "https://deno.land/std@0.153.0/encoding/base64.ts" \ No newline at end of file diff --git a/src/handlers/count.ts b/src/handlers/count.ts new file mode 100644 index 0000000..4274ece --- /dev/null +++ b/src/handlers/count.ts @@ -0,0 +1,3 @@ +export function count (_: URLPatternResult): Response { + throw new Error("not implemented"); +} \ No newline at end of file diff --git a/src/handlers/notImplemented.ts b/src/handlers/notImplemented.ts new file mode 100644 index 0000000..d9b1d1c --- /dev/null +++ b/src/handlers/notImplemented.ts @@ -0,0 +1,3 @@ +export function notImplemented (_request: Request, _match: URLPatternResult): Response { + throw new Error("not implemented"); +} diff --git a/src/helpers/processParams.ts b/src/helpers/processParams.ts new file mode 100644 index 0000000..a2566a5 --- /dev/null +++ b/src/helpers/processParams.ts @@ -0,0 +1,88 @@ +import { getTagTypes } from "../meta/finder.ts"; + +function extractQuery(request: Request) { + const url = new URL(request.url); + const query: Record = {}; + for (const param of url.searchParams) { + if (query[param[0]]) { + query[param[0]].push(param[1]); + } else query[param[0]] = new Array(param[1]); + } + return query; +} + +type ParamTag = { + label: string; + min?: string; + max?: string; + type?: string; + maxIsExclusive?: boolean; +}; + +type Params = { + tags?: ParamTag[]; + nottags?: ParamTag[]; + text?: string; + limit?: number; + offset?: number; +}; + +export async function processParams(request: Request): Promise { + const query = extractQuery(request); + const result: Params = {}; + const tags = query.tag?.map((t) => t.split(";")) ?? []; + const nottags = query.nottag?.map((t) => t.split(";")) ?? []; + result.text = query.text?.[0]; + result.limit = parseInt(query.limit?.[0], 10) > 0 + ? parseInt(query.limit[0]) + : undefined; + result.offset = parseInt(query.offset?.[0], 10) >= 0 + ? parseInt(query.offset[0]) + : undefined; + return await getTagTypes( + tags.map((e) => e[0]).concat(nottags.map((e) => e[0])), + ).then((types) => { + function tagMap(t: string[]): ParamTag { + const label = t[0]; + const type = types.find((e) => e[0] === t[0])?.[1]; + let min = t[1]; + let max = t[2]; + let maxIsExclusive; + if (type === "http://www.w3.org/2001/XMLSchema#date") { + if (min) { + switch (min.length) { + case 4: + min += "-01-01"; + break; + case 7: + min += "-01"; + break; + } + } + if (max) { + switch (max.length) { + case 4: + max += "-12-31"; + break; + case 7: { + const month = parseInt(max.substring(5), 10) + 1; + if (month < 13) { + max = max.substring(0, 5) + "-" + + month.toString().padStart(2, "0") + "-01"; + maxIsExclusive = true; + } else { + max += "-31"; + } + break; + } + } + } + } + return { label, min, max, type, maxIsExclusive }; + } + result.tags = tags.map(tagMap); + result.nottags = nottags.map(tagMap); + console.log("eh??", result); + return result; + }); +} diff --git a/src/main.ts b/src/main.ts new file mode 100644 index 0000000..e75b254 --- /dev/null +++ b/src/main.ts @@ -0,0 +1,12 @@ +import { serve } from "./server/server.ts"; + +console.log("Starting tridoc backend server"); + +// TODO Check external dependencies + +if (!Deno.env.get("TRIDOC_PWD")) { + throw new Error("No password set"); +} + +serve(); +console.log("Tridoc backend server is listening on port 8000"); diff --git a/src/meta/finder.ts b/src/meta/finder.ts new file mode 100644 index 0000000..2c1b13d --- /dev/null +++ b/src/meta/finder.ts @@ -0,0 +1,20 @@ +export async function getTagTypes(labels: string[]): Promise { + const response=await fetch("http://fuseki:3030/3DOC/query",{ + method: "POST", + headers: { +"Authorization": "Basic "+btoa("admin:pw123"), +"Content-Type": "application/sparql-query", + }, + body: `PREFIX tridoc: +SELECT DISTINCT ?l ?t WHERE { VALUES ?l { "${labels.join('" "')}" } ?s tridoc:label ?l . OPTIONAL { ?s tridoc:valueType ?t . } }`, +}); +const json=await response.json(); +return json.results.bindings.map((binding: Record) => { + const result_1=[]; + result_1[0]=binding.l.value; + if(binding.t) { +result_1[1]=binding.t.value; + } + return result_1; +}); +} diff --git a/src/server/routes.ts b/src/server/routes.ts new file mode 100644 index 0000000..63e0866 --- /dev/null +++ b/src/server/routes.ts @@ -0,0 +1,85 @@ +import { notImplemented } from "../handlers/notImplemented.ts"; + +export const routes: { + [method: string]: { + pattern: URLPattern; + handler: (request: Request, match: URLPatternResult) => Response; + }[]; +} = { + "GET": [{ + pattern: new URLPattern({ pathname: "/count" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/comment" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/tag" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/thumb" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/title" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/meta" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/raw/rdf" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/raw/zip" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/raw/tgz" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/tag" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/tag/:tagLabel" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/version" }), + handler: notImplemented, + }], + "POST": [{ + pattern: new URLPattern({ pathname: "/doc" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/comment" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/tag" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/tag" }), + handler: notImplemented, + }], + "PUT": [{ + pattern: new URLPattern({ pathname: "/doc/:id/title" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/raw/zip" }), + handler: notImplemented, + }], + "DELETE": [{ + pattern: new URLPattern({ pathname: "/doc/:id" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/tag/:tagLabel" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/doc/:id/title" }), + handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/tag/:tagLabel" }), + handler: notImplemented, + }], +}; diff --git a/src/server/server.ts b/src/server/server.ts new file mode 100644 index 0000000..02833b8 --- /dev/null +++ b/src/server/server.ts @@ -0,0 +1,52 @@ +import { encode, serve as stdServe } from "../deps.ts"; +import { routes } from "./routes.ts"; + +const isAuthenticated = (request: Request) => { + return request.headers.get("Authorization") === + "Basic " + encode("tridoc:" + Deno.env.get("TRIDOC_PWD")); +}; + +const handler = (request: Request): Response => { + const path = request.url.slice(request.url.indexOf("/", "https://".length)); + console.log((new Date()).toISOString(), request.method, path); + try { + if (!isAuthenticated(request)) { + console.log( + (new Date()).toISOString(), + request.method, + path, + "→ 401: Not Authenticated", + ); + return new Response("401: Not Authenticated", { + status: 401, + headers: { "WWW-Authenticate": "Basic" }, + }); + } + + const route = routes[request.method]?.find(({ pattern }) => + pattern.test(request.url) + ); + if (route) { + return route.handler(request, route.pattern.exec(request.url)!); + } + + console.log( + (new Date()).toISOString(), + request.method, + path, + "→ 404: Path not foun", + ); + return new Response("404: Path not found", { status: 404 }); + } catch (error) { + console.log( + (new Date()).toISOString(), + request.method, + path, + "→ 500:", + error, + ); + return new Response("500: " + error, { status: 500 }); + } +}; + +export const serve = () => stdServe(handler, { onListen: undefined }); From 5d136365c2dde7e980ccf1af077d68525878b0df Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 15:14:58 +0200 Subject: [PATCH 03/35] fmt --- src/deps.ts | 2 +- src/handlers/count.ts | 4 ++-- src/handlers/notImplemented.ts | 5 ++++- src/meta/finder.ts | 32 ++++++++++++++++++-------------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/deps.ts b/src/deps.ts index 1ad18b0..b75190d 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,2 +1,2 @@ export { serve } from "https://deno.land/std@0.153.0/http/mod.ts"; -export { encode } from "https://deno.land/std@0.153.0/encoding/base64.ts" \ No newline at end of file +export { encode } from "https://deno.land/std@0.153.0/encoding/base64.ts"; diff --git a/src/handlers/count.ts b/src/handlers/count.ts index 4274ece..586ddf0 100644 --- a/src/handlers/count.ts +++ b/src/handlers/count.ts @@ -1,3 +1,3 @@ -export function count (_: URLPatternResult): Response { +export function count(_: URLPatternResult): Response { throw new Error("not implemented"); -} \ No newline at end of file +} diff --git a/src/handlers/notImplemented.ts b/src/handlers/notImplemented.ts index d9b1d1c..65e2a35 100644 --- a/src/handlers/notImplemented.ts +++ b/src/handlers/notImplemented.ts @@ -1,3 +1,6 @@ -export function notImplemented (_request: Request, _match: URLPatternResult): Response { +export function notImplemented( + _request: Request, + _match: URLPatternResult, +): Response { throw new Error("not implemented"); } diff --git a/src/meta/finder.ts b/src/meta/finder.ts index 2c1b13d..b271513 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -1,20 +1,24 @@ export async function getTagTypes(labels: string[]): Promise { - const response=await fetch("http://fuseki:3030/3DOC/query",{ + const response = await fetch("http://fuseki:3030/3DOC/query", { method: "POST", headers: { -"Authorization": "Basic "+btoa("admin:pw123"), -"Content-Type": "application/sparql-query", + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", }, body: `PREFIX tridoc: -SELECT DISTINCT ?l ?t WHERE { VALUES ?l { "${labels.join('" "')}" } ?s tridoc:label ?l . OPTIONAL { ?s tridoc:valueType ?t . } }`, -}); -const json=await response.json(); -return json.results.bindings.map((binding: Record) => { - const result_1=[]; - result_1[0]=binding.l.value; - if(binding.t) { -result_1[1]=binding.t.value; - } - return result_1; -}); +SELECT DISTINCT ?l ?t WHERE { VALUES ?l { "${ + labels.join('" "') + }" } ?s tridoc:label ?l . OPTIONAL { ?s tridoc:valueType ?t . } }`, + }); + const json = await response.json(); + return json.results.bindings.map( + (binding: Record) => { + const result_1 = []; + result_1[0] = binding.l.value; + if (binding.t) { + result_1[1] = binding.t.value; + } + return result_1; + }, + ); } From 315819c92020b85fa89095d642ebb3832e587b22 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 15:27:23 +0200 Subject: [PATCH 04/35] added GET /version --- src/deps.ts | 2 ++ src/handlers/version.ts | 8 ++++++++ src/server/routes.ts | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 src/handlers/version.ts diff --git a/src/deps.ts b/src/deps.ts index b75190d..e422717 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,2 +1,4 @@ +export const VERSION = "1.6.0-alpha.deno"; + export { serve } from "https://deno.land/std@0.153.0/http/mod.ts"; export { encode } from "https://deno.land/std@0.153.0/encoding/base64.ts"; diff --git a/src/handlers/version.ts b/src/handlers/version.ts new file mode 100644 index 0000000..edf10bf --- /dev/null +++ b/src/handlers/version.ts @@ -0,0 +1,8 @@ +import { VERSION } from "../deps.ts"; + +export function version( + _request: Request, + _match: URLPatternResult, +): Response { + return new Response(VERSION); +} diff --git a/src/server/routes.ts b/src/server/routes.ts index 63e0866..3afd997 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -1,4 +1,5 @@ import { notImplemented } from "../handlers/notImplemented.ts"; +import { version } from "../handlers/version.ts"; export const routes: { [method: string]: { @@ -47,7 +48,7 @@ export const routes: { handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/version" }), - handler: notImplemented, + handler: version, }], "POST": [{ pattern: new URLPattern({ pathname: "/doc" }), From b11dee5536b6661fe772371e0e7862a976b85425 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 16:10:16 +0200 Subject: [PATCH 05/35] docker framework --- old/.dockerignore => .dockerignore | 1 + old/.gitignore => .gitignore | 0 old/DEV-README.md => DEV-README.md | 0 Dockerfile | 18 +++++++++++++++ old/config-tdb.ttl => config-tdb.ttl | 0 ...cker-compose.yml => dev-docker-compose.yml | 0 old/docker-cmd.sh => docker-cmd.sh | 3 +-- old/docker-compose.yml => docker-compose.yml | 0 old/Dockerfile | 11 ---------- old/package.json | 22 ------------------- 10 files changed, 20 insertions(+), 35 deletions(-) rename old/.dockerignore => .dockerignore (88%) rename old/.gitignore => .gitignore (100%) rename old/DEV-README.md => DEV-README.md (100%) create mode 100644 Dockerfile rename old/config-tdb.ttl => config-tdb.ttl (100%) rename old/dev-docker-compose.yml => dev-docker-compose.yml (100%) rename old/docker-cmd.sh => docker-cmd.sh (80%) rename old/docker-compose.yml => docker-compose.yml (100%) delete mode 100644 old/Dockerfile delete mode 100644 old/package.json diff --git a/old/.dockerignore b/.dockerignore similarity index 88% rename from old/.dockerignore rename to .dockerignore index f145ba1..e911418 100644 --- a/old/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +old blobs fuseki-base node_modules \ No newline at end of file diff --git a/old/.gitignore b/.gitignore similarity index 100% rename from old/.gitignore rename to .gitignore diff --git a/old/DEV-README.md b/DEV-README.md similarity index 100% rename from old/DEV-README.md rename to DEV-README.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b983ae9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM denoland/deno:1.26.2 + +EXPOSE 8000 + +RUN mkdir -p /usr/src/app/src +WORKDIR /usr/src/app + +RUN apt update \ + && apt -y install pdfsandwich tesseract-ocr-deu tesseract-ocr-fra curl +RUN rm /etc/ImageMagick-6/policy.xml + +USER deno +COPY src/deps.ts src/deps.ts +RUN deno cache src/deps.ts + +COPY . . + +CMD [ "/bin/bash", "/usr/src/app/docker-cmd.sh" ] \ No newline at end of file diff --git a/old/config-tdb.ttl b/config-tdb.ttl similarity index 100% rename from old/config-tdb.ttl rename to config-tdb.ttl diff --git a/old/dev-docker-compose.yml b/dev-docker-compose.yml similarity index 100% rename from old/dev-docker-compose.yml rename to dev-docker-compose.yml diff --git a/old/docker-cmd.sh b/docker-cmd.sh similarity index 80% rename from old/docker-cmd.sh rename to docker-cmd.sh index 56e8c9d..4ef74cc 100644 --- a/old/docker-cmd.sh +++ b/docker-cmd.sh @@ -1,10 +1,9 @@ #!/bin/bash -sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' set -m -yarn start & +deno run --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts & sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ diff --git a/old/docker-compose.yml b/docker-compose.yml similarity index 100% rename from old/docker-compose.yml rename to docker-compose.yml diff --git a/old/Dockerfile b/old/Dockerfile deleted file mode 100644 index 3fbabeb..0000000 --- a/old/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM node:lts-buster -EXPOSE 8000 -RUN apt update \ - && apt -y install pdfsandwich tesseract-ocr-deu tesseract-ocr-fra -RUN rm /etc/ImageMagick-6/policy.xml -RUN mkdir -p /usr/src/app -WORKDIR /usr/src/app -COPY . /usr/src/app -RUN yarn install -RUN chmod +x /usr/src/app/docker-cmd.sh -CMD [ "/usr/src/app/docker-cmd.sh" ] \ No newline at end of file diff --git a/old/package.json b/old/package.json deleted file mode 100644 index 5a603ea..0000000 --- a/old/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "tridoc-backend", - "version": "1.5.2", - "description": "Simple RDF-Based Document Management System", - "main": "lib/server", - "repository": "git@github.com:tridoc/tridoc-backend.git", - "author": "Noam Bachmann ", - "license": "MIT", - "dependencies": { - "adm-zip": "^0.4.16", - "archiver": "^3.1.1", - "hapi": "^17.5.2", - "hapi-auth-basic": "^5.0.0", - "nanoid": "^1.1.0", - "node-fetch": "^2.2.0", - "pdfjs-dist": "^2.0.489" - }, - "scripts": { - "start": "node lib/server.js", - "start-with-pwd": "TRIDOC_PWD='tridoc' node lib/server.js" - } -} From 08ff7a3e53b7af341efcd27e0eb3ba82335f7bef Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 15:16:37 +0000 Subject: [PATCH 06/35] setup devcontainer --- .devcontainer/Dockerfile | 20 +++++++++++ .devcontainer/devcontainer.json | 47 ++++++++++++++++++++++++ .devcontainer/docker-cmd.sh | 11 ++++++ .devcontainer/docker-compose.yml | 37 +++++++++++++++++++ .vscode/settings.json | 2 +- DEV-README.md | 62 +++----------------------------- 6 files changed, 120 insertions(+), 59 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-cmd.sh create mode 100644 .devcontainer/docker-compose.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..7a9c183 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,20 @@ +FROM denoland/deno:1.26.2 + +EXPOSE 8000 + +RUN mkdir -p /home/deno +RUN chown -R deno /home/deno +RUN mkdir -p /usr/src/app/src +WORKDIR /usr/src/app + +RUN apt update \ + && apt -y install pdfsandwich tesseract-ocr-deu tesseract-ocr-fra curl git +RUN rm /etc/ImageMagick-6/policy.xml + +USER deno +COPY src/deps.ts src/deps.ts +RUN deno cache src/deps.ts + +COPY . . + +CMD [ "/bin/bash", "/usr/src/app/.devcontainer/docker-cmd.sh" ] \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..03ce50f --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,47 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/docker-existing-docker-compose +// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml. +{ + "name": "Existing Docker Compose (Extend)", + + // Update the 'dockerComposeFile' list if you have more compose files or use different names. + // The .devcontainer/docker-compose.yml file contains any overrides you need/want to make. + "dockerComposeFile": [ + "../dev-docker-compose.yml", + "docker-compose.yml" + ], + + "containerEnv": { + "TRIDOC_PWD": "pw123", + }, + + // The 'service' property is the name of the service for the container that VS Code should + // use. Update this value and .devcontainer/docker-compose.yml to the real service name. + "service": "tridoc", + + // The optional 'workspaceFolder' property is the path VS Code should open by default when + // connected. This is typically a file mount in .devcontainer/docker-compose.yml + "workspaceFolder": "/usr/src/app", + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line if you want start specific services in your Docker Compose config. + "runServices": [ "fuseki" ], + + // Uncomment the next line if you want to keep your containers running after VS Code shuts down. + // "shutdownAction": "none", + + // Uncomment the next line to run commands after the container is created - for example installing curl. + // "postCreateCommand": "apt-get update && apt-get install -y curl", + + // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "deno", + "customizations": { + "vscode": { + "extensions": [ + "denoland.vscode-deno" + ] + } + } +} diff --git a/.devcontainer/docker-cmd.sh b/.devcontainer/docker-cmd.sh new file mode 100644 index 0000000..a1d8c86 --- /dev/null +++ b/.devcontainer/docker-cmd.sh @@ -0,0 +1,11 @@ +#!/bin/bash +echo 'Attempting to create Dataset "3DOC"' +curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ + -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' +set -m +deno run --watch --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts & +sleep 5 +echo 'Attempting to create Dataset "3DOC"' +curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ + -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' +fg 1 diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..0638fa6 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' +services: + # Update this to the name of the service you want to work with in your docker-compose.yml file + tridoc: + # If you want add a non-root user to your Dockerfile, you can use the "remoteUser" + # property in devcontainer.json to cause VS Code its sub-processes (terminals, tasks, + # debugging) to execute as the user. Uncomment the next line if you want the entire + # container to run as this user instead. Note that, on Linux, you may need to + # ensure the UID and GID of the container user you create matches your local user. + # See https://aka.ms/vscode-remote/containers/non-root for details. + # + user: deno + + # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer + # folder. Note that the path of the Dockerfile and context is relative to the *primary* + # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" + # array). The sample below assumes your primary file is in the root of your project. + # + build: + context: . + dockerfile: .devcontainer/Dockerfile + + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - .:/usr/src/app:cached + + # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details. + # - /var/run/docker.sock:/var/run/docker.sock + + # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust. + # cap_add: + # - SYS_PTRACE + # security_opt: + # - seccomp:unconfined + + # Overrides default command so things don't shut down after the process ends. + # command: "/bin/bash -c \"TRIDOC_PWD=\\\"pw123\\\" deno run --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts &\\\n sleep 5\\\n echo 'Attempting to create Dataset \\\"3DOC\\\"'\\\n curl 'http://fuseki:3030/$/datasets' -H \\\"Authorization: Basic $(echo -n admin:pw123 | base64)\\\" -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb'\\\n fg 1\\\n /bin/sh -c \\\"while sleep 1000; do :; done\\\"\"" diff --git a/.vscode/settings.json b/.vscode/settings.json index 85ece8f..1535e13 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "npm.packageManager": "yarn", "deno.enable": true, + "deno.lint": true, "deno.unstable": true } \ No newline at end of file diff --git a/DEV-README.md b/DEV-README.md index 8ddc74e..f809fd1 100644 --- a/DEV-README.md +++ b/DEV-README.md @@ -1,65 +1,11 @@ # tridoc -## Table Of Contents - * [Easy Setup with Docker-Compose](#easy-setup-with-docker-compose) - * [Dev Build](#dev-build) - * [Production Build](#production-build) - * [Setup with Persistent Fuseki](#setup-with-persistent-fuseki) - * [Docker](#docker) - * [Manual](#manual) +## Run "live" -## Developer Guide +Use the vscode-devcontainer: this will start tridoc and fuseki. -This assumes a Unix/Linux/wsl system with bash - -### Easy Setup with Docker-Compose - -This will setup tridoc on port 8000 and fuseki avaliable on port 8001. - -Replace `YOUR PASSWORD HERE` in the first command with your choice of password. - -#### Dev Build: - -``` -export TRIDOC_PWD="YOUR PASSWORD HERE" -docker-compose -f dev-docker-compose.yml build -docker-compose -f dev-docker-compose.yml up -``` - -#### Production Build: - -``` -export TRIDOC_PWD="YOUR PASSWORD HERE" -docker-compose build -docker-compose up -``` - -### Setup with Persistent Fuseki - -The following method expect an instance of Fuseki running on http://fuseki:3030/ with user `admin` and password `pw123`. This fuseki instance must have lucene indexing enabled and configured as in [config-tdb.ttl](config-tdb.ttl). - -#### Docker: - -``` -docker build -t tridoc . -docker run -p 8000:8000 -e TRIDOC_PWD="YOUR PASSWORD HERE" tridoc -``` - -#### Manual: - -Install the following dependencies: - -``` -node:12.18 yarn pdfsandwich tesseract-ocr-deu tesseract-ocr-fra -``` - -And run the following commands - -``` -rm /etc/ImageMagick-6/policy.xml -yarn install -bash docker-cmd.sh -``` +It will use TRIDOC_PWD = "pw123". +Access tridoc from http://localhost:8000 and fuseki from http://localhost:8001 ## Tips & Tricks From f08e7f1d0975fcf539c3598ac1ee6ae88933b979 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 18:34:20 +0000 Subject: [PATCH 07/35] added zip restore --- .devcontainer/Dockerfile | 2 +- .devcontainer/docker-cmd.sh | 2 +- Dockerfile | 2 +- README.md | 9 +++---- deno.jsonc | 4 ++-- docker-cmd.sh | 2 +- src/deps.ts | 6 +++-- src/handlers/notImplemented.ts | 2 +- src/handlers/raw.ts | 44 ++++++++++++++++++++++++++++++++++ src/handlers/version.ts | 4 ++-- src/meta/store.ts | 14 +++++++++++ src/server/routes.ts | 8 +++++-- src/server/server.ts | 6 ++--- 13 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 src/handlers/raw.ts create mode 100644 src/meta/store.ts diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7a9c183..39c4d3a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -8,7 +8,7 @@ RUN mkdir -p /usr/src/app/src WORKDIR /usr/src/app RUN apt update \ - && apt -y install pdfsandwich tesseract-ocr-deu tesseract-ocr-fra curl git + && apt -y install pdfsandwich tesseract-ocr-deu tesseract-ocr-fra curl git zip unzip RUN rm /etc/ImageMagick-6/policy.xml USER deno diff --git a/.devcontainer/docker-cmd.sh b/.devcontainer/docker-cmd.sh index a1d8c86..d48095a 100644 --- a/.devcontainer/docker-cmd.sh +++ b/.devcontainer/docker-cmd.sh @@ -3,7 +3,7 @@ echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' set -m -deno run --watch --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts & +deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts & sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ diff --git a/Dockerfile b/Dockerfile index b983ae9..680ee6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN mkdir -p /usr/src/app/src WORKDIR /usr/src/app RUN apt update \ - && apt -y install pdfsandwich tesseract-ocr-deu tesseract-ocr-fra curl + && apt -y install pdfsandwich tesseract-ocr-deu tesseract-ocr-fra curl zip unzip RUN rm /etc/ImageMagick-6/policy.xml USER deno diff --git a/README.md b/README.md index 0965e93..ebff991 100644 --- a/README.md +++ b/README.md @@ -105,8 +105,8 @@ When getting a comment, a JSON array with objects of the following structure is ## API -| Address | Method | Description | Request / Payload | Response | Implemented in Version | -| - | - | - | - | - | - | +| Address | Method | Description | Request / Payload | Response | Implemented in Version | deno? | +| - | - | - | - | - | - | - | | `/count` | GET | Count (matching) documents | [1](#f1) [3](#f3) | Number | 1.1.0 | | `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | | `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | @@ -123,13 +123,14 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc/{id}/title` | DELETE | Reset document title | - | - | 1.1.0 | | `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | | `/raw/rdf` | GET | Get all metadata as RDF. Useful for Backups | [4](#f4) | RDF, Content-Type defined over request Headers or ?accept. Fallback to text/turtle. | 1.1.0 | +| `/raw/rdf` | DELETE | "Cancel" failed zip upload—use only if certain it’s done & failed | | | (deno only) | ✅ | | `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | -| `/raw/zip` | PUT | Replace all data with backup zip | ZIP | Replaces the metadata and adds the blobs from the zip | 1.3.0 | +| `/raw/zip` | PUT | Replace all data with backup zip | ZIP | Replaces the metadata and adds the blobs from the zip | 1.3.0 | ✅ | | `/tag` | POST | Create new tag | See above | - | 1.1.0 | | `/tag` | GET | Get (list of) all tags | - | - | 1.1.0 | | `/tag/{tagLabel}` | GET | Get Documents with this tag. Same as `/doc?tag={tagLabel}` | [1](#f1) [2](#f2) | Array of objects with document identifiers and titles (where available) | 1.1.0 | | `/tag/{tagLabel}` | DELETE | Delete this tag | - | - | 1.1.0 | -| `/version` | GET | Get tridoc version | - | semver version number | 1.1.0 | +| `/version` | GET | Get tridoc version | - | semver version number | 1.1.0 | ✅ | #### URL-Parameters supported: diff --git a/deno.jsonc b/deno.jsonc index 86149a3..085b9a3 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -5,7 +5,7 @@ } }, "tasks": { - "run": "deno run --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts", - "run-watch": "deno run --watch --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts" + "run": "deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttls --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts", + "run-watch": "deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts" } } diff --git a/docker-cmd.sh b/docker-cmd.sh index 4ef74cc..4bb7aa7 100644 --- a/docker-cmd.sh +++ b/docker-cmd.sh @@ -3,7 +3,7 @@ echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' set -m -deno run --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts & +deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts & sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ diff --git a/src/deps.ts b/src/deps.ts index e422717..209487c 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,4 +1,6 @@ export const VERSION = "1.6.0-alpha.deno"; -export { serve } from "https://deno.land/std@0.153.0/http/mod.ts"; -export { encode } from "https://deno.land/std@0.153.0/encoding/base64.ts"; +export { encode } from "https://deno.land/std@0.160.0/encoding/base64.ts"; +export { emptyDir } from "https://deno.land/std@0.160.0/fs/mod.ts"; +export { serve } from "https://deno.land/std@0.160.0/http/mod.ts"; +export { writableStreamFromWriter } from "https://deno.land/std@0.160.0/streams/mod.ts"; diff --git a/src/handlers/notImplemented.ts b/src/handlers/notImplemented.ts index 65e2a35..10da909 100644 --- a/src/handlers/notImplemented.ts +++ b/src/handlers/notImplemented.ts @@ -1,6 +1,6 @@ export function notImplemented( _request: Request, _match: URLPatternResult, -): Response { +): Promise { throw new Error("not implemented"); } diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts new file mode 100644 index 0000000..a69dfe7 --- /dev/null +++ b/src/handlers/raw.ts @@ -0,0 +1,44 @@ +import { emptyDir, writableStreamFromWriter } from "../deps.ts"; +import { restore } from "../meta/store.ts"; + +const decoder = new TextDecoder("utf-8"); + +export async function deleteRdfFile( + _request: Request, + _match: URLPatternResult, +): Promise { + await Deno.remove("rdf.ttl"); + return new Response("200: OK"); +} + +export async function putZip( + request: Request, + _match: URLPatternResult, +): Promise { + try { + await Deno.stat("rdf.ttl"); + throw new Error( + "Can't unzip concurrently: rdf.ttl already exists. If you know what you are doing, clear this message with HTTP DELETE /raw/rdf", + ); + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) { + throw error; + } + } + await emptyDir("blobs"); + const zipPath = "blobs/zip-" + Date.now(); + const zip = await Deno.open(zipPath, { write: true, create: true }); + const writableStream = writableStreamFromWriter(zip); + await request.body?.pipeTo(writableStream); + const p = Deno.run({ cmd: ["unzip", zipPath] }); + const { success, code } = await p.status(); + if (success) { + await Deno.remove(zipPath); + const turtleData = decoder.decode(await Deno.readFile("rdf.ttl")); + await Deno.remove("rdf.ttl"); + await restore(turtleData); + return new Response("200: OK"); + } else { + throw new Error("unzip failed with code " + code); + } +} diff --git a/src/handlers/version.ts b/src/handlers/version.ts index edf10bf..a092472 100644 --- a/src/handlers/version.ts +++ b/src/handlers/version.ts @@ -3,6 +3,6 @@ import { VERSION } from "../deps.ts"; export function version( _request: Request, _match: URLPatternResult, -): Response { - return new Response(VERSION); +): Promise { + return new Promise((resolve) => resolve(new Response(VERSION))); } diff --git a/src/meta/store.ts b/src/meta/store.ts new file mode 100644 index 0000000..808c6c7 --- /dev/null +++ b/src/meta/store.ts @@ -0,0 +1,14 @@ +export function restore(turtleData: string) { + const statement = `CLEAR GRAPH ; + INSERT DATA { + GRAPH { ${turtleData} } + }`; + return fetch("http://fuseki:3030/3DOC/update", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-update", + }, + body: statement, + }); +} diff --git a/src/server/routes.ts b/src/server/routes.ts index 3afd997..45169cb 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -1,10 +1,11 @@ import { notImplemented } from "../handlers/notImplemented.ts"; +import { deleteRdfFile, putZip } from "../handlers/raw.ts"; import { version } from "../handlers/version.ts"; export const routes: { [method: string]: { pattern: URLPattern; - handler: (request: Request, match: URLPatternResult) => Response; + handler: (request: Request, match: URLPatternResult) => Promise; }[]; } = { "GET": [{ @@ -68,7 +69,7 @@ export const routes: { handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/raw/zip" }), - handler: notImplemented, + handler: putZip, }], "DELETE": [{ pattern: new URLPattern({ pathname: "/doc/:id" }), @@ -82,5 +83,8 @@ export const routes: { }, { pattern: new URLPattern({ pathname: "/tag/:tagLabel" }), handler: notImplemented, + }, { + pattern: new URLPattern({ pathname: "/raw/rdf" }), + handler: deleteRdfFile, }], }; diff --git a/src/server/server.ts b/src/server/server.ts index 02833b8..876303e 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -6,7 +6,7 @@ const isAuthenticated = (request: Request) => { "Basic " + encode("tridoc:" + Deno.env.get("TRIDOC_PWD")); }; -const handler = (request: Request): Response => { +const handler = async (request: Request): Promise => { const path = request.url.slice(request.url.indexOf("/", "https://".length)); console.log((new Date()).toISOString(), request.method, path); try { @@ -27,14 +27,14 @@ const handler = (request: Request): Response => { pattern.test(request.url) ); if (route) { - return route.handler(request, route.pattern.exec(request.url)!); + return await route.handler(request, route.pattern.exec(request.url)!); } console.log( (new Date()).toISOString(), request.method, path, - "→ 404: Path not foun", + "→ 404: Path not found", ); return new Response("404: Path not found", { status: 404 }); } catch (error) { From 1ed70b6d9c096b9791037b107e2f5282bc18795f Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 19:08:55 +0000 Subject: [PATCH 08/35] added cors headers --- src/handlers/cors.ts | 12 ++++++++++++ src/handlers/raw.ts | 5 +++-- src/helpers/cors.ts | 11 +++++++++++ src/server/routes.ts | 5 +++++ src/server/server.ts | 11 ++++++----- 5 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 src/handlers/cors.ts create mode 100644 src/helpers/cors.ts diff --git a/src/handlers/cors.ts b/src/handlers/cors.ts new file mode 100644 index 0000000..cf3d819 --- /dev/null +++ b/src/handlers/cors.ts @@ -0,0 +1,12 @@ +import { respond } from "../helpers/cors.ts"; + +export function options( + _request: Request, + _match: URLPatternResult, +): Promise { + return new Promise((resolve) => + resolve( + respond(undefined, { status: 204 }) + ) + ); +} \ No newline at end of file diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts index a69dfe7..1845781 100644 --- a/src/handlers/raw.ts +++ b/src/handlers/raw.ts @@ -1,4 +1,5 @@ import { emptyDir, writableStreamFromWriter } from "../deps.ts"; +import { respond } from "../helpers/cors.ts"; import { restore } from "../meta/store.ts"; const decoder = new TextDecoder("utf-8"); @@ -8,7 +9,7 @@ export async function deleteRdfFile( _match: URLPatternResult, ): Promise { await Deno.remove("rdf.ttl"); - return new Response("200: OK"); + return respond("200: OK"); } export async function putZip( @@ -37,7 +38,7 @@ export async function putZip( const turtleData = decoder.decode(await Deno.readFile("rdf.ttl")); await Deno.remove("rdf.ttl"); await restore(turtleData); - return new Response("200: OK"); + return respond("200: OK"); } else { throw new Error("unzip failed with code " + code); } diff --git a/src/helpers/cors.ts b/src/helpers/cors.ts new file mode 100644 index 0000000..ac7b85e --- /dev/null +++ b/src/helpers/cors.ts @@ -0,0 +1,11 @@ +export function respond(body?: BodyInit, init?: ResponseInit) { + return new Response(body, { + ...init, + headers: { + ...init?.headers, + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "POST, PUT, DELETE, GET, OPTIONS", + "Access-Control-Allow-Headers": "Authorization", + }, + }); +} diff --git a/src/server/routes.ts b/src/server/routes.ts index 45169cb..82a123b 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -1,3 +1,4 @@ +import { options } from "../handlers/cors.ts"; import { notImplemented } from "../handlers/notImplemented.ts"; import { deleteRdfFile, putZip } from "../handlers/raw.ts"; import { version } from "../handlers/version.ts"; @@ -8,6 +9,10 @@ export const routes: { handler: (request: Request, match: URLPatternResult) => Promise; }[]; } = { + "OPTIONS": [{ + pattern: new URLPattern({ pathname: "*" }), + handler: options, + }], "GET": [{ pattern: new URLPattern({ pathname: "/count" }), handler: notImplemented, diff --git a/src/server/server.ts b/src/server/server.ts index 876303e..03f31e8 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -1,8 +1,9 @@ import { encode, serve as stdServe } from "../deps.ts"; +import { respond } from "../helpers/cors.ts"; import { routes } from "./routes.ts"; const isAuthenticated = (request: Request) => { - return request.headers.get("Authorization") === + return (request.method === "OPTIONS") || request.headers.get("Authorization") === "Basic " + encode("tridoc:" + Deno.env.get("TRIDOC_PWD")); }; @@ -17,7 +18,7 @@ const handler = async (request: Request): Promise => { path, "→ 401: Not Authenticated", ); - return new Response("401: Not Authenticated", { + return respond("401: Not Authenticated", { status: 401, headers: { "WWW-Authenticate": "Basic" }, }); @@ -36,16 +37,16 @@ const handler = async (request: Request): Promise => { path, "→ 404: Path not found", ); - return new Response("404: Path not found", { status: 404 }); + return respond("404: Path not found", { status: 404 }); } catch (error) { console.log( (new Date()).toISOString(), request.method, path, - "→ 500:", + "→ 500: ", error, ); - return new Response("500: " + error, { status: 500 }); + return respond("500: " + error, { status: 500 }); } }; From d208e9e5d1ba167098f6d6968da97d327bd7f694 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 19:08:55 +0000 Subject: [PATCH 09/35] added cors headers --- src/handlers/cors.ts | 12 ++++++++++++ src/handlers/raw.ts | 5 +++-- src/handlers/version.ts | 3 ++- src/helpers/cors.ts | 11 +++++++++++ src/server/routes.ts | 5 +++++ src/server/server.ts | 11 ++++++----- 6 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 src/handlers/cors.ts create mode 100644 src/helpers/cors.ts diff --git a/src/handlers/cors.ts b/src/handlers/cors.ts new file mode 100644 index 0000000..cf3d819 --- /dev/null +++ b/src/handlers/cors.ts @@ -0,0 +1,12 @@ +import { respond } from "../helpers/cors.ts"; + +export function options( + _request: Request, + _match: URLPatternResult, +): Promise { + return new Promise((resolve) => + resolve( + respond(undefined, { status: 204 }) + ) + ); +} \ No newline at end of file diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts index a69dfe7..1845781 100644 --- a/src/handlers/raw.ts +++ b/src/handlers/raw.ts @@ -1,4 +1,5 @@ import { emptyDir, writableStreamFromWriter } from "../deps.ts"; +import { respond } from "../helpers/cors.ts"; import { restore } from "../meta/store.ts"; const decoder = new TextDecoder("utf-8"); @@ -8,7 +9,7 @@ export async function deleteRdfFile( _match: URLPatternResult, ): Promise { await Deno.remove("rdf.ttl"); - return new Response("200: OK"); + return respond("200: OK"); } export async function putZip( @@ -37,7 +38,7 @@ export async function putZip( const turtleData = decoder.decode(await Deno.readFile("rdf.ttl")); await Deno.remove("rdf.ttl"); await restore(turtleData); - return new Response("200: OK"); + return respond("200: OK"); } else { throw new Error("unzip failed with code " + code); } diff --git a/src/handlers/version.ts b/src/handlers/version.ts index a092472..2864385 100644 --- a/src/handlers/version.ts +++ b/src/handlers/version.ts @@ -1,8 +1,9 @@ import { VERSION } from "../deps.ts"; +import { respond } from "../helpers/cors.ts"; export function version( _request: Request, _match: URLPatternResult, ): Promise { - return new Promise((resolve) => resolve(new Response(VERSION))); + return new Promise((resolve) => resolve(respond(VERSION))); } diff --git a/src/helpers/cors.ts b/src/helpers/cors.ts new file mode 100644 index 0000000..ac7b85e --- /dev/null +++ b/src/helpers/cors.ts @@ -0,0 +1,11 @@ +export function respond(body?: BodyInit, init?: ResponseInit) { + return new Response(body, { + ...init, + headers: { + ...init?.headers, + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "POST, PUT, DELETE, GET, OPTIONS", + "Access-Control-Allow-Headers": "Authorization", + }, + }); +} diff --git a/src/server/routes.ts b/src/server/routes.ts index 45169cb..82a123b 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -1,3 +1,4 @@ +import { options } from "../handlers/cors.ts"; import { notImplemented } from "../handlers/notImplemented.ts"; import { deleteRdfFile, putZip } from "../handlers/raw.ts"; import { version } from "../handlers/version.ts"; @@ -8,6 +9,10 @@ export const routes: { handler: (request: Request, match: URLPatternResult) => Promise; }[]; } = { + "OPTIONS": [{ + pattern: new URLPattern({ pathname: "*" }), + handler: options, + }], "GET": [{ pattern: new URLPattern({ pathname: "/count" }), handler: notImplemented, diff --git a/src/server/server.ts b/src/server/server.ts index 876303e..03f31e8 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -1,8 +1,9 @@ import { encode, serve as stdServe } from "../deps.ts"; +import { respond } from "../helpers/cors.ts"; import { routes } from "./routes.ts"; const isAuthenticated = (request: Request) => { - return request.headers.get("Authorization") === + return (request.method === "OPTIONS") || request.headers.get("Authorization") === "Basic " + encode("tridoc:" + Deno.env.get("TRIDOC_PWD")); }; @@ -17,7 +18,7 @@ const handler = async (request: Request): Promise => { path, "→ 401: Not Authenticated", ); - return new Response("401: Not Authenticated", { + return respond("401: Not Authenticated", { status: 401, headers: { "WWW-Authenticate": "Basic" }, }); @@ -36,16 +37,16 @@ const handler = async (request: Request): Promise => { path, "→ 404: Path not found", ); - return new Response("404: Path not found", { status: 404 }); + return respond("404: Path not found", { status: 404 }); } catch (error) { console.log( (new Date()).toISOString(), request.method, path, - "→ 500:", + "→ 500: ", error, ); - return new Response("500: " + error, { status: 500 }); + return respond("500: " + error, { status: 500 }); } }; From ffce65fc0141ce9adbd5741eff6ecc81e5bae20c Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 19:30:13 +0000 Subject: [PATCH 10/35] added count --- src/handlers/cors.ts | 4 +- src/handlers/count.ts | 13 +++++- src/helpers/processParams.ts | 12 +++--- src/meta/finder.ts | 76 ++++++++++++++++++++++++++++++++++++ src/server/routes.ts | 3 +- src/server/server.ts | 5 ++- 6 files changed, 100 insertions(+), 13 deletions(-) diff --git a/src/handlers/cors.ts b/src/handlers/cors.ts index cf3d819..d98c74a 100644 --- a/src/handlers/cors.ts +++ b/src/handlers/cors.ts @@ -6,7 +6,7 @@ export function options( ): Promise { return new Promise((resolve) => resolve( - respond(undefined, { status: 204 }) + respond(undefined, { status: 204 }), ) ); -} \ No newline at end of file +} diff --git a/src/handlers/count.ts b/src/handlers/count.ts index 586ddf0..06d0f26 100644 --- a/src/handlers/count.ts +++ b/src/handlers/count.ts @@ -1,3 +1,12 @@ -export function count(_: URLPatternResult): Response { - throw new Error("not implemented"); +import { respond } from "../helpers/cors.ts"; +import { processParams } from "../helpers/processParams.ts"; +import { getDocumentNumber } from "../meta/finder.ts"; + +export async function count( + request: Request, + _match: URLPatternResult, +): Promise { + const params = await processParams(request); + const count = await getDocumentNumber(params); + return respond("" + count); } diff --git a/src/helpers/processParams.ts b/src/helpers/processParams.ts index a2566a5..d02de40 100644 --- a/src/helpers/processParams.ts +++ b/src/helpers/processParams.ts @@ -12,14 +12,14 @@ function extractQuery(request: Request) { } type ParamTag = { - label: string; - min?: string; - max?: string; - type?: string; - maxIsExclusive?: boolean; + label: string; // [0] + min?: string; // [1] + max?: string; // [2] + type?: string; // [3] + maxIsExclusive?: boolean; //[5] }; -type Params = { +export type Params = { tags?: ParamTag[]; nottags?: ParamTag[]; text?: string; diff --git a/src/meta/finder.ts b/src/meta/finder.ts index b271513..a7cb6d5 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -1,3 +1,79 @@ +import { Params } from "../helpers/processParams.ts"; + +export async function getDocumentNumber( + { tags = [], nottags = [], text = "" }: Params, +) { + let tagQuery = ""; + for (let i = 0; i < tags.length; i++) { + if (tags[i].type) { + tagQuery += `{ ?s tridoc:tag ?ptag${i} . + ?ptag${i} tridoc:parameterizableTag ?atag${i} . + ?ptag${i} tridoc:value ?v${i} . + ?atag${i} tridoc:label "${tags[i].label}" . + ${ + tags[i].min + ? `FILTER (?v${i} >= "${tags[i].min}"^^<${tags[i].type}> )` + : "" + } + ${ + tags[i].max + ? `FILTER (?v${i} ${tags[i].maxIsExclusive ? "<" : "<="} "${ + tags[i].max + }"^^<${tags[i].type}> )` + : "" + } }`; + } else { + tagQuery += `{ ?s tridoc:tag ?tag${i} . + ?tag${i} tridoc:label "${tags[i].label}" . }`; + } + } + for (let i = 0; i < nottags.length; i++) { + if (nottags[i].type) { + tagQuery += `FILTER NOT EXISTS { ?s tridoc:tag ?ptag${i} . + ?ptag${i} tridoc:parameterizableTag ?atag${i} . + ?ptag${i} tridoc:value ?v${i} . + ?atag${i} tridoc:label "${nottags[i].label}" . + ${ + nottags[i].min + ? `FILTER (?v${i} >= "${nottags[i].min}"^^<${nottags[i].type}> )` + : "" + } + ${ + nottags[i].max + ? `FILTER (?v${i} ${nottags[i].maxIsExclusive ? "<" : "<="} "${ + nottags[i].max + }"^^<${nottags[i].type}> )` + : "" + } }`; + } else { + tagQuery += `FILTER NOT EXISTS { ?s tridoc:tag ?tag${i} . + ?tag${i} tridoc:label "${nottags[i].label}" . }`; + } + } + return await fetch("http://fuseki:3030/3DOC/query", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", + }, + body: "PREFIX rdf: \n" + + "PREFIX s: \n" + + "PREFIX tridoc: \n" + + "PREFIX text: \n" + + "SELECT (COUNT(DISTINCT ?s) as ?count)\n" + + "WHERE {\n" + + " ?s s:identifier ?identifier .\n" + + tagQuery + + (text + ? '{ { ?s text:query (s:name "' + text + + '") } UNION { ?s text:query (s:text "' + text + '")} } .\n' + : "") + + "}", + }).then((response) => response.json()).then((json) => + json.results.bindings[0].count.value as number + ); +} + export async function getTagTypes(labels: string[]): Promise { const response = await fetch("http://fuseki:3030/3DOC/query", { method: "POST", diff --git a/src/server/routes.ts b/src/server/routes.ts index 82a123b..41f046d 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -1,4 +1,5 @@ import { options } from "../handlers/cors.ts"; +import { count } from "../handlers/count.ts"; import { notImplemented } from "../handlers/notImplemented.ts"; import { deleteRdfFile, putZip } from "../handlers/raw.ts"; import { version } from "../handlers/version.ts"; @@ -15,7 +16,7 @@ export const routes: { }], "GET": [{ pattern: new URLPattern({ pathname: "/count" }), - handler: notImplemented, + handler: count, }, { pattern: new URLPattern({ pathname: "/doc" }), handler: notImplemented, diff --git a/src/server/server.ts b/src/server/server.ts index 03f31e8..d49f105 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -3,8 +3,9 @@ import { respond } from "../helpers/cors.ts"; import { routes } from "./routes.ts"; const isAuthenticated = (request: Request) => { - return (request.method === "OPTIONS") || request.headers.get("Authorization") === - "Basic " + encode("tridoc:" + Deno.env.get("TRIDOC_PWD")); + return (request.method === "OPTIONS") || + request.headers.get("Authorization") === + "Basic " + encode("tridoc:" + Deno.env.get("TRIDOC_PWD")); }; const handler = async (request: Request): Promise => { From 4c4586667af9b56038d248f722d7003e116f694e Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 19:41:33 +0000 Subject: [PATCH 11/35] added count --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ebff991..94c9412 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ When getting a comment, a JSON array with objects of the following structure is | Address | Method | Description | Request / Payload | Response | Implemented in Version | deno? | | - | - | - | - | - | - | - | -| `/count` | GET | Count (matching) documents | [1](#f1) [3](#f3) | Number | 1.1.0 | +| `/count` | GET | Count (matching) documents | [1](#f1) [3](#f3) | Number | 1.1.0 | ✅ | | `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | | `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | | `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | From 9a72686c9d5d1561b9d988b9e79a3315db5e3a61 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 20:20:49 +0000 Subject: [PATCH 12/35] added GET /doc/:id --- README.md | 2 +- src/handlers/doc.ts | 36 +++++++++++++++ src/meta/finder.ts | 102 ++++++++++++++++++++++++++++++++++++++++++- src/server/routes.ts | 11 ++--- 4 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 src/handlers/doc.ts diff --git a/README.md b/README.md index 94c9412..94486d5 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/count` | GET | Count (matching) documents | [1](#f1) [3](#f3) | Number | 1.1.0 | ✅ | | `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | | `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | -| `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | +| `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | ✅ | | `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | | `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | | `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts new file mode 100644 index 0000000..7a273e2 --- /dev/null +++ b/src/handlers/doc.ts @@ -0,0 +1,36 @@ +import { respond } from "../helpers/cors.ts"; +import { processParams } from "../helpers/processParams.ts"; +import { getDocumentList } from "../meta/finder.ts"; + +function getPath(id: string) { + return "./blobs/" + id.slice(0, 2) + "/" + id.slice(2, 6) + "/" + + id.slice(6, 14) + "/" + id; +} + +export async function getPDF( + _request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + const path = getPath(id); + try { + const file = await Deno.open(path, { read: true }); + // Build a readable stream so the file doesn't have to be fully loaded into memory while we send it + const readableStream = file.readable; + return respond(readableStream); + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) { + return respond("404 Not Found", { status: 404 }); + } + throw error; + } +} + +export async function list( + request: Request, + _match: URLPatternResult, +): Promise { + const params = await processParams(request); + const response = await getDocumentList(params); + return respond(JSON.stringify(response)); +} diff --git a/src/meta/finder.ts b/src/meta/finder.ts index a7cb6d5..46c70c6 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -1,7 +1,107 @@ import { Params } from "../helpers/processParams.ts"; +/** takes: { tags: [string, string, string][], nottags: [string, string, string][], text: string, limit: number, offset: number } */ +export async function getDocumentList( + { tags = [], nottags = [], text, limit, offset }: Params, +) { + let tagQuery = ""; + for (let i = 0; i < tags.length; i++) { + if (tags[i].type) { + tagQuery += `{ ?s tridoc:tag ?ptag${i} . + ?ptag${i} tridoc:parameterizableTag ?atag${i} . + ?ptag${i} tridoc:value ?v${i} . + ?atag${i} tridoc:label "${tags[i].label}" . + ${ + tags[i].min + ? `FILTER (?v${i} >= "${tags[i].min}"^^<${tags[i].type}> )` + : "" + } + ${ + tags[i].max + ? `FILTER (?v${i} ${tags[i].maxIsExclusive ? "<" : "<="} "${ + tags[i].max + }"^^<${tags[i].type}> )` + : "" + } }`; + } else { + tagQuery += `{ ?s tridoc:tag ?tag${i} . + ?tag${i} tridoc:label "${tags[i].label}" . }`; + } + } + for (let i = 0; i < nottags.length; i++) { + if (nottags[i].type) { + tagQuery += `FILTER NOT EXISTS { ?s tridoc:tag ?ptag${i} . + ?ptag${i} tridoc:parameterizableTag ?atag${i} . + ?ptag${i} tridoc:value ?v${i} . + ?atag${i} tridoc:label "${nottags[i].label}" . + ${ + nottags[i].min + ? `FILTER (?v${i} >= "${nottags[i].min}"^^<${nottags[i].type}> )` + : "" + } + ${ + nottags[i].max + ? `FILTER (?v${i} ${nottags[i].maxIsExclusive ? "<" : "<="} "${ + nottags[i].max + }"^^<${nottags[i].type}> )` + : "" + } }`; + } else { + tagQuery += `FILTER NOT EXISTS { ?s tridoc:tag ?tag${i} . + ?tag${i} tridoc:label "${nottags[i].label}" . }`; + } + } + const body = "PREFIX rdf: \n" + + "PREFIX s: \n" + + "PREFIX tridoc: \n" + + "PREFIX text: \n" + + "SELECT DISTINCT ?s ?identifier ?title ?date\n" + + "WHERE {\n" + + " ?s s:identifier ?identifier .\n" + + " ?s s:dateCreated ?date .\n" + + tagQuery + + " OPTIONAL { ?s s:name ?title . }\n" + + (text + ? '{ { ?s text:query (s:name "' + text + + '") } UNION { ?s text:query (s:text "' + text + '")} } .\n' + : "") + + "}\n" + + "ORDER BY desc(?date)\n" + + (limit ? "LIMIT " + limit + "\n" : "") + + (offset ? "OFFSET " + offset : ""); + return await fetch("http://fuseki:3030/3DOC/query", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", + }, + body: body, + }).then((response) => response.json()).then((json) => + json.results.bindings.map( + ( + binding: { + s: { value: string }; + identifier: { value: string }; + title?: { value: string }; + date?: { value: string }; + }, + ) => { + const result: Record = {}; + result.identifier = binding.identifier.value; + if (binding.title) { + result.title = binding.title.value; + } + if (binding.date) { + result.created = binding.date.value; + } + return result; + }, + ) + ); +} + export async function getDocumentNumber( - { tags = [], nottags = [], text = "" }: Params, + { tags = [], nottags = [], text }: Params, ) { let tagQuery = ""; for (let i = 0; i < tags.length; i++) { diff --git a/src/server/routes.ts b/src/server/routes.ts index 41f046d..107bd22 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -1,7 +1,8 @@ import { options } from "../handlers/cors.ts"; import { count } from "../handlers/count.ts"; +import * as doc from "../handlers/doc.ts"; import { notImplemented } from "../handlers/notImplemented.ts"; -import { deleteRdfFile, putZip } from "../handlers/raw.ts"; +import * as raw from "../handlers/raw.ts"; import { version } from "../handlers/version.ts"; export const routes: { @@ -19,10 +20,10 @@ export const routes: { handler: count, }, { pattern: new URLPattern({ pathname: "/doc" }), - handler: notImplemented, + handler: doc.list, }, { pattern: new URLPattern({ pathname: "/doc/:id" }), - handler: notImplemented, + handler: doc.getPDF, }, { pattern: new URLPattern({ pathname: "/doc/:id/comment" }), handler: notImplemented, @@ -75,7 +76,7 @@ export const routes: { handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/raw/zip" }), - handler: putZip, + handler: raw.putZip, }], "DELETE": [{ pattern: new URLPattern({ pathname: "/doc/:id" }), @@ -91,6 +92,6 @@ export const routes: { handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/raw/rdf" }), - handler: deleteRdfFile, + handler: raw.deleteRdfFile, }], }; From c1c0b30584177844e1a13a5133e027d1c579a419 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sat, 22 Oct 2022 20:50:34 +0000 Subject: [PATCH 13/35] added GET /doc/:id/comment --- src/handlers/doc.ts | 13 +++++++++++-- src/meta/finder.ts | 42 +++++++++++++++++++++++++++++++++++------- src/server/routes.ts | 2 +- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 7a273e2..eb28665 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -1,12 +1,21 @@ import { respond } from "../helpers/cors.ts"; import { processParams } from "../helpers/processParams.ts"; -import { getDocumentList } from "../meta/finder.ts"; +import * as metafinder from "../meta/finder.ts"; function getPath(id: string) { return "./blobs/" + id.slice(0, 2) + "/" + id.slice(2, 6) + "/" + id.slice(6, 14) + "/" + id; } +export async function getComments( + _request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + const response = await metafinder.getComments(id); + return respond(JSON.stringify(response)); +} + export async function getPDF( _request: Request, match: URLPatternResult, @@ -31,6 +40,6 @@ export async function list( _match: URLPatternResult, ): Promise { const params = await processParams(request); - const response = await getDocumentList(params); + const response = await metafinder.getDocumentList(params); return respond(JSON.stringify(response)); } diff --git a/src/meta/finder.ts b/src/meta/finder.ts index 46c70c6..8a7dab2 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -1,6 +1,39 @@ import { Params } from "../helpers/processParams.ts"; -/** takes: { tags: [string, string, string][], nottags: [string, string, string][], text: string, limit: number, offset: number } */ +export async function getComments(id: string) { + const query = `PREFIX rdf: +PREFIX xsd: +PREFIX tridoc: +PREFIX s: +SELECT DISTINCT ?d ?t WHERE { + GRAPH { + s:comment [ + a s:Comment ; + s:dateCreated ?d ; + s:text ?t + ] . + } +}`; + return await fetch("http://fuseki:3030/3DOC/query", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", + }, + body: query, + }).then((response) => { + if (response.ok) { + return response.json(); + } else { + throw new Error("" + response); + } + }).then((json) => + json.results.bindings.map((binding: Record) => { + return { text: binding.t.value, created: binding.d.value }; + }) + ); +} + export async function getDocumentList( { tags = [], nottags = [], text, limit, offset }: Params, ) { @@ -79,12 +112,7 @@ export async function getDocumentList( }).then((response) => response.json()).then((json) => json.results.bindings.map( ( - binding: { - s: { value: string }; - identifier: { value: string }; - title?: { value: string }; - date?: { value: string }; - }, + binding: Record, ) => { const result: Record = {}; result.identifier = binding.identifier.value; diff --git a/src/server/routes.ts b/src/server/routes.ts index 107bd22..8f40997 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -26,7 +26,7 @@ export const routes: { handler: doc.getPDF, }, { pattern: new URLPattern({ pathname: "/doc/:id/comment" }), - handler: notImplemented, + handler: doc.getComments, }, { pattern: new URLPattern({ pathname: "/doc/:id/tag" }), handler: notImplemented, From 1c5795ce9e2014feae0d68da9da1a23a6e7fb958 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 09:37:10 +0000 Subject: [PATCH 14/35] added POST /doc --- .devcontainer/docker-cmd.sh | 2 +- .devcontainer/docker-compose.yml | 2 +- DEV-README.md | 9 +++++ README.md | 6 ++-- deno.jsonc | 4 +-- docker-cmd.sh | 2 +- src/deps.ts | 4 ++- src/handlers/doc.ts | 62 ++++++++++++++++++++++++++++++++ src/handlers/raw.ts | 15 ++++---- src/helpers/cors.ts | 2 +- src/helpers/pdfprocessor.ts | 8 +++++ src/meta/store.ts | 40 +++++++++++++++++++++ src/server/routes.ts | 2 +- 13 files changed, 138 insertions(+), 20 deletions(-) create mode 100644 src/helpers/pdfprocessor.ts diff --git a/.devcontainer/docker-cmd.sh b/.devcontainer/docker-cmd.sh index d48095a..cfe9532 100644 --- a/.devcontainer/docker-cmd.sh +++ b/.devcontainer/docker-cmd.sh @@ -3,7 +3,7 @@ echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' set -m -deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts & +deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts & sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 0638fa6..3b042ba 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -34,4 +34,4 @@ services: # - seccomp:unconfined # Overrides default command so things don't shut down after the process ends. - # command: "/bin/bash -c \"TRIDOC_PWD=\\\"pw123\\\" deno run --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD src/main.ts &\\\n sleep 5\\\n echo 'Attempting to create Dataset \\\"3DOC\\\"'\\\n curl 'http://fuseki:3030/$/datasets' -H \\\"Authorization: Basic $(echo -n admin:pw123 | base64)\\\" -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb'\\\n fg 1\\\n /bin/sh -c \\\"while sleep 1000; do :; done\\\"\"" + # command: "/bin/bash -c \"TRIDOC_PWD=\\\"pw123\\\" deno run --allow-net --allow-read=blobs --allow-write=blobs --allow-run=convert,pdfsandwich --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts &\\\n sleep 5\\\n echo 'Attempting to create Dataset \\\"3DOC\\\"'\\\n curl 'http://fuseki:3030/$/datasets' -H \\\"Authorization: Basic $(echo -n admin:pw123 | base64)\\\" -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb'\\\n fg 1\\\n /bin/sh -c \\\"while sleep 1000; do :; done\\\"\"" diff --git a/DEV-README.md b/DEV-README.md index f809fd1..cc6a6a2 100644 --- a/DEV-README.md +++ b/DEV-README.md @@ -7,6 +7,15 @@ Use the vscode-devcontainer: this will start tridoc and fuseki. It will use TRIDOC_PWD = "pw123". Access tridoc from http://localhost:8000 and fuseki from http://localhost:8001 +You might need to `chown deno:deno` blobs/ and fuseki-base (attach bash to docker as root from outside) + +Watch the logs from outside of vscode with + +```sh +docker logs -f tridoc-backend_tridoc_1 +``` + + ## Tips & Tricks - Upload Backups with diff --git a/README.md b/README.md index 94486d5..621a8ec 100644 --- a/README.md +++ b/README.md @@ -108,12 +108,12 @@ When getting a comment, a JSON array with objects of the following structure is | Address | Method | Description | Request / Payload | Response | Implemented in Version | deno? | | - | - | - | - | - | - | - | | `/count` | GET | Count (matching) documents | [1](#f1) [3](#f3) | Number | 1.1.0 | ✅ | -| `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | -| `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | +| `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | ✅ | +| `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | | `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | ✅ | | `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | | `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | -| `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | +| `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | ✅ | | `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | | `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | | `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 | diff --git a/deno.jsonc b/deno.jsonc index 085b9a3..5413c78 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -5,7 +5,7 @@ } }, "tasks": { - "run": "deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttls --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts", - "run-watch": "deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts" + "run": "deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttls --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts", + "run-watch": "deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts" } } diff --git a/docker-cmd.sh b/docker-cmd.sh index 4bb7aa7..0780872 100644 --- a/docker-cmd.sh +++ b/docker-cmd.sh @@ -3,7 +3,7 @@ echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' set -m -deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,zip,unzip --allow-env=TRIDOC_PWD src/main.ts & +deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts & sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ diff --git a/src/deps.ts b/src/deps.ts index 209487c..c2af758 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,6 +1,8 @@ export const VERSION = "1.6.0-alpha.deno"; export { encode } from "https://deno.land/std@0.160.0/encoding/base64.ts"; -export { emptyDir } from "https://deno.land/std@0.160.0/fs/mod.ts"; +export { emptyDir, ensureDir } from "https://deno.land/std@0.160.0/fs/mod.ts"; export { serve } from "https://deno.land/std@0.160.0/http/mod.ts"; export { writableStreamFromWriter } from "https://deno.land/std@0.160.0/streams/mod.ts"; + +export { nanoid } from "https://deno.land/x/nanoid@v3.0.0/mod.ts"; diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index eb28665..cf54c0e 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -1,12 +1,29 @@ +import { ensureDir } from "https://deno.land/std@0.160.0/fs/ensure_dir.ts"; +import { nanoid, writableStreamFromWriter } from "../deps.ts"; import { respond } from "../helpers/cors.ts"; +import { getText } from "../helpers/pdfprocessor.ts"; import { processParams } from "../helpers/processParams.ts"; import * as metafinder from "../meta/finder.ts"; +import * as metastore from "../meta/store.ts"; + +function getDir(id: string) { + return "./blobs/" + id.slice(0, 2) + "/" + id.slice(2, 6) + "/" + + id.slice(6, 14); +} function getPath(id: string) { return "./blobs/" + id.slice(0, 2) + "/" + id.slice(2, 6) + "/" + id.slice(6, 14) + "/" + id; } +function datecheck(request: Request) { + const url = new URL(request.url); + const regex = + /^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-6]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-6]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-6]\d([+-][0-2]\d:[0-5]\d|Z))$/; + const date = url.searchParams.get("date"); + return date ? (regex.test(date) ? date : undefined) : undefined; +} + export async function getComments( _request: Request, match: URLPatternResult, @@ -43,3 +60,48 @@ export async function list( const response = await metafinder.getDocumentList(params); return respond(JSON.stringify(response)); } + +export async function postPDF( + request: Request, + _match: URLPatternResult, +): Promise { + const id = nanoid(); + const path = getPath(id); + await ensureDir(getDir(id)); + const pdf = await Deno.open(path, { write: true, create: true }); + const writableStream = writableStreamFromWriter(pdf); + await request.body?.pipeTo(writableStream); + console.log((new Date()).toISOString(), "Document created with id", id); + let text = await getText(path); + if (text.length < 4) { + // run OCR + const lang = Deno.env.get("OCR_LANG") || "fra+deu+eng"; + const p = Deno.run({ cmd: ["pdfsandwich", "-rgb", "-lang", lang, path] }); + const { success, code } = await p.status(); + if (!success) throw new Error("pdfsandwich failed with code " + code); + // pdfsandwich generates a file with the same name + _ocr + await Deno.rename(path + "_ocr", path); + text = await getText(path); + console.log((new Date()).toISOString(), id, ": OCR finished"); + } + // no await as we don’t care for the result - if it fails, the thumbnail will be created upon request. + Deno.run({ + cmd: [ + "convert", + "-thumbnail", + "300x", + "-alpha", + "remove", + `${path}[0]`, + `${path}.png`, + ], + }); + const date = datecheck(request); + await metastore.storeDocument({ id, text, date }); + return respond(undefined, { + headers: { + "Location": "/doc/" + id, + "Access-Control-Expose-Headers": "Location", + }, + }); +} diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts index 1845781..c819489 100644 --- a/src/handlers/raw.ts +++ b/src/handlers/raw.ts @@ -33,13 +33,10 @@ export async function putZip( await request.body?.pipeTo(writableStream); const p = Deno.run({ cmd: ["unzip", zipPath] }); const { success, code } = await p.status(); - if (success) { - await Deno.remove(zipPath); - const turtleData = decoder.decode(await Deno.readFile("rdf.ttl")); - await Deno.remove("rdf.ttl"); - await restore(turtleData); - return respond("200: OK"); - } else { - throw new Error("unzip failed with code " + code); - } + if (!success) throw new Error("unzip failed with code " + code); + await Deno.remove(zipPath); + const turtleData = decoder.decode(await Deno.readFile("rdf.ttl")); + await Deno.remove("rdf.ttl"); + await restore(turtleData); + return respond("200: OK"); } diff --git a/src/helpers/cors.ts b/src/helpers/cors.ts index ac7b85e..907f2ad 100644 --- a/src/helpers/cors.ts +++ b/src/helpers/cors.ts @@ -5,7 +5,7 @@ export function respond(body?: BodyInit, init?: ResponseInit) { ...init?.headers, "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "POST, PUT, DELETE, GET, OPTIONS", - "Access-Control-Allow-Headers": "Authorization", + "Access-Control-Allow-Headers": "Authorization, Content-Type", }, }); } diff --git a/src/helpers/pdfprocessor.ts b/src/helpers/pdfprocessor.ts new file mode 100644 index 0000000..585b3db --- /dev/null +++ b/src/helpers/pdfprocessor.ts @@ -0,0 +1,8 @@ +const decoder = new TextDecoder("utf-8"); + +export async function getText(path: string) { + const p = Deno.run({ cmd: ["pdftotext", path, "-"], stdout: "piped" }); + const { success, code } = await p.status(); + if (!success) throw new Error("pdfsandwich failed with code " + code); + return decoder.decode(await p.output()); +} diff --git a/src/meta/store.ts b/src/meta/store.ts index 808c6c7..9b23424 100644 --- a/src/meta/store.ts +++ b/src/meta/store.ts @@ -1,3 +1,10 @@ +function escapeLiteral(string: string) { + return string.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace( + /\r/g, + "\\r", + ).replace(/'/g, "\\'").replace(/"/g, '\\"'); +} + export function restore(turtleData: string) { const statement = `CLEAR GRAPH ; INSERT DATA { @@ -12,3 +19,36 @@ export function restore(turtleData: string) { body: statement, }); } + +export async function storeDocument( + { id, text, date }: { id: string; text: string; date?: string }, +) { + const created = (date ? new Date(date) : new Date()).toISOString(); + const query = ` +PREFIX rdf: +PREFIX xsd: +PREFIX s: +INSERT DATA { + GRAPH { + rdf:type s:DigitalDocument ; + s:dateCreated "${created}"^^xsd:dateTime ; + s:identifier "${id}" ; + s:text "${escapeLiteral(text)}" . + } +}`; + return await fetch("http://fuseki:3030/3DOC/update", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-update", + }, + body: query, + }).then((response) => { + //console.log("Fuseki returned: "+response.status); + if (response.ok) { + return response; + } else { + throw new Error("Error from Fuseki: " + response.statusText); + } + }); +} diff --git a/src/server/routes.ts b/src/server/routes.ts index 8f40997..aeafbb0 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -60,7 +60,7 @@ export const routes: { }], "POST": [{ pattern: new URLPattern({ pathname: "/doc" }), - handler: notImplemented, + handler: doc.postPDF, }, { pattern: new URLPattern({ pathname: "/doc/:id/comment" }), handler: notImplemented, From 36ec78cfdf0926cb8b995950a78e594c0bf8a3af Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 09:57:48 +0000 Subject: [PATCH 15/35] added filename to GET /doc --- src/handlers/doc.ts | 10 ++++++- src/meta/finder.ts | 68 ++++++++++++++++++++++++++++++++------------- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index cf54c0e..660bb1f 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -40,10 +40,18 @@ export async function getPDF( const id = match.pathname.groups.id; const path = getPath(id); try { + const fileName = await metafinder.getBasicMeta(id).then(( + { title, created }, + ) => title || created || "document"); const file = await Deno.open(path, { read: true }); // Build a readable stream so the file doesn't have to be fully loaded into memory while we send it const readableStream = file.readable; - return respond(readableStream); + return respond(readableStream, { + headers: { + "content-disposition": `inline; filename="${encodeURI(fileName)}.pdf"`, + "content-type": "application/pdf", + }, + }); } catch (error) { if (!(error instanceof Deno.errors.NotFound)) { return respond("404 Not Found", { status: 404 }); diff --git a/src/meta/finder.ts b/src/meta/finder.ts index 8a7dab2..fba26e8 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -1,5 +1,14 @@ import { Params } from "../helpers/processParams.ts"; +type SparqlJson = { + head: { + vars: string[]; + }; + results: { + bindings: { [key: string]: { type: string; value: string } }[]; + }; +}; + export async function getComments(id: string) { const query = `PREFIX rdf: PREFIX xsd: @@ -27,8 +36,8 @@ SELECT DISTINCT ?d ?t WHERE { } else { throw new Error("" + response); } - }).then((json) => - json.results.bindings.map((binding: Record) => { + }).then((json: SparqlJson) => + json.results.bindings.map((binding) => { return { text: binding.t.value, created: binding.d.value }; }) ); @@ -109,22 +118,18 @@ export async function getDocumentList( "Content-Type": "application/sparql-query", }, body: body, - }).then((response) => response.json()).then((json) => - json.results.bindings.map( - ( - binding: Record, - ) => { - const result: Record = {}; - result.identifier = binding.identifier.value; - if (binding.title) { - result.title = binding.title.value; - } - if (binding.date) { - result.created = binding.date.value; - } - return result; - }, - ) + }).then((response) => response.json()).then((json: SparqlJson) => + json.results.bindings.map((binding) => { + const result: Record = {}; + result.identifier = binding.identifier.value; + if (binding.title) { + result.title = binding.title.value; + } + if (binding.date) { + result.created = binding.date.value; + } + return result; + }) ); } @@ -197,11 +202,34 @@ export async function getDocumentNumber( '") } UNION { ?s text:query (s:text "' + text + '")} } .\n' : "") + "}", - }).then((response) => response.json()).then((json) => - json.results.bindings[0].count.value as number + }).then((response) => response.json()).then((json: SparqlJson) => + parseInt(json.results.bindings[0].count.value, 10) ); } +export async function getBasicMeta(id: string) { + return await fetch("http://fuseki:3030/3DOC/query", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", + }, + body: "PREFIX rdf: \n" + + "PREFIX s: \n" + + "SELECT ?title ?date\n" + + "WHERE {\n" + + ' ?s s:identifier "' + id + '" .\n' + + " ?s s:dateCreated ?date .\n" + + " OPTIONAL { ?s s:name ?title . }\n" + + "}", + }).then((response) => response.json()).then((json: SparqlJson) => { + return { + title: json.results.bindings[0].title?.value, + created: json.results.bindings[0].date?.value, + }; + }); +} + export async function getTagTypes(labels: string[]): Promise { const response = await fetch("http://fuseki:3030/3DOC/query", { method: "POST", From 1ef5b307a38b04ec5efe4fd6e5a870bb1c0502ac Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 10:18:44 +0000 Subject: [PATCH 16/35] added delete --- README.md | 2 +- src/handlers/doc.ts | 10 ++++++++++ src/meta/delete.ts | 8 ++++++++ src/meta/finder.ts | 4 ++-- src/meta/fusekiFetch.ts | 26 ++++++++++++++++++++++++++ src/server/routes.ts | 2 +- 6 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 src/meta/delete.ts create mode 100644 src/meta/fusekiFetch.ts diff --git a/README.md b/README.md index 621a8ec..7664b31 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | ✅ | | `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | | `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | ✅ | -| `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | +| `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | ✅ | | `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | | `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | ✅ | | `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 660bb1f..9319fa6 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -3,6 +3,7 @@ import { nanoid, writableStreamFromWriter } from "../deps.ts"; import { respond } from "../helpers/cors.ts"; import { getText } from "../helpers/pdfprocessor.ts"; import { processParams } from "../helpers/processParams.ts"; +import * as metadelete from "../meta/delete.ts"; import * as metafinder from "../meta/finder.ts"; import * as metastore from "../meta/store.ts"; @@ -24,6 +25,15 @@ function datecheck(request: Request) { return date ? (regex.test(date) ? date : undefined) : undefined; } +export async function deleteDoc( + _request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + await metadelete.deleteFile(id); + return respond(undefined, { status: 204 }); +} + export async function getComments( _request: Request, match: URLPatternResult, diff --git a/src/meta/delete.ts b/src/meta/delete.ts new file mode 100644 index 0000000..46cd3aa --- /dev/null +++ b/src/meta/delete.ts @@ -0,0 +1,8 @@ +import { fusekiFetch } from "./fusekiFetch.ts"; + +export function deleteFile(id: string) { + return fusekiFetch(` +WITH +DELETE { ?p ?o } +WHERE { ?p ?o }`); +} diff --git a/src/meta/finder.ts b/src/meta/finder.ts index fba26e8..f8ac358 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -224,8 +224,8 @@ export async function getBasicMeta(id: string) { "}", }).then((response) => response.json()).then((json: SparqlJson) => { return { - title: json.results.bindings[0].title?.value, - created: json.results.bindings[0].date?.value, + title: json.results.bindings[0]?.title?.value, + created: json.results.bindings[0]?.date?.value, }; }); } diff --git a/src/meta/fusekiFetch.ts b/src/meta/fusekiFetch.ts new file mode 100644 index 0000000..bc607a8 --- /dev/null +++ b/src/meta/fusekiFetch.ts @@ -0,0 +1,26 @@ +type SparqlJson = { + head: { + vars: string[]; + }; + results: { + bindings: { [key: string]: { type: string; value: string } }[]; + }; +}; + +export async function fusekiFetch(query: string): Promise { + console.log((new Date()).toISOString(), "→ FUSEKI", query); + return await fetch("http://fuseki:3030/3DOC/query", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", + }, + body: query, + }).then((response) => { + if (response.ok) { + return response.json(); + } else { + throw new Error("Fuseki Error: " + response); + } + }); +} diff --git a/src/server/routes.ts b/src/server/routes.ts index aeafbb0..5fed9e9 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -80,7 +80,7 @@ export const routes: { }], "DELETE": [{ pattern: new URLPattern({ pathname: "/doc/:id" }), - handler: notImplemented, + handler: doc.deleteDoc, }, { pattern: new URLPattern({ pathname: "/doc/:id/tag/:tagLabel" }), handler: notImplemented, From 47285319a656bbdfcfb811302f39a3e148f37d87 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 10:41:12 +0000 Subject: [PATCH 17/35] redactored to fusekiFetch/Update instead of fetch --- src/meta/delete.ts | 4 +- src/meta/finder.ts | 113 ++++++++++++---------------------------- src/meta/fusekiFetch.ts | 22 ++++++-- src/meta/store.ts | 35 +++---------- 4 files changed, 61 insertions(+), 113 deletions(-) diff --git a/src/meta/delete.ts b/src/meta/delete.ts index 46cd3aa..9398486 100644 --- a/src/meta/delete.ts +++ b/src/meta/delete.ts @@ -1,7 +1,7 @@ -import { fusekiFetch } from "./fusekiFetch.ts"; +import { fusekiUpdate } from "./fusekiFetch.ts"; export function deleteFile(id: string) { - return fusekiFetch(` + return fusekiUpdate(` WITH DELETE { ?p ?o } WHERE { ?p ?o }`); diff --git a/src/meta/finder.ts b/src/meta/finder.ts index f8ac358..15bdf63 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -1,13 +1,5 @@ import { Params } from "../helpers/processParams.ts"; - -type SparqlJson = { - head: { - vars: string[]; - }; - results: { - bindings: { [key: string]: { type: string; value: string } }[]; - }; -}; +import { fusekiFetch } from "./fusekiFetch.ts"; export async function getComments(id: string) { const query = `PREFIX rdf: @@ -23,20 +15,7 @@ SELECT DISTINCT ?d ?t WHERE { ] . } }`; - return await fetch("http://fuseki:3030/3DOC/query", { - method: "POST", - headers: { - "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-query", - }, - body: query, - }).then((response) => { - if (response.ok) { - return response.json(); - } else { - throw new Error("" + response); - } - }).then((json: SparqlJson) => + return await fusekiFetch(query).then((json) => json.results.bindings.map((binding) => { return { text: binding.t.value, created: binding.d.value }; }) @@ -111,14 +90,7 @@ export async function getDocumentList( "ORDER BY desc(?date)\n" + (limit ? "LIMIT " + limit + "\n" : "") + (offset ? "OFFSET " + offset : ""); - return await fetch("http://fuseki:3030/3DOC/query", { - method: "POST", - headers: { - "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-query", - }, - body: body, - }).then((response) => response.json()).then((json: SparqlJson) => + return await fusekiFetch(body).then((json) => json.results.bindings.map((binding) => { const result: Record = {}; result.identifier = binding.identifier.value; @@ -183,46 +155,32 @@ export async function getDocumentNumber( ?tag${i} tridoc:label "${nottags[i].label}" . }`; } } - return await fetch("http://fuseki:3030/3DOC/query", { - method: "POST", - headers: { - "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-query", - }, - body: "PREFIX rdf: \n" + - "PREFIX s: \n" + - "PREFIX tridoc: \n" + - "PREFIX text: \n" + - "SELECT (COUNT(DISTINCT ?s) as ?count)\n" + - "WHERE {\n" + - " ?s s:identifier ?identifier .\n" + - tagQuery + - (text - ? '{ { ?s text:query (s:name "' + text + - '") } UNION { ?s text:query (s:text "' + text + '")} } .\n' - : "") + - "}", - }).then((response) => response.json()).then((json: SparqlJson) => - parseInt(json.results.bindings[0].count.value, 10) - ); + return await fusekiFetch(` +PREFIX rdf: +PREFIX s: +PREFIX tridoc: +PREFIX text: +SELECT (COUNT(DISTINCT ?s) as ?count) +WHERE { + ?s s:identifier ?identifier . + ${tagQuery} + ${ + text + ? `{ { ?s text:query (s:name "${text}") } UNION { ?s text:query (s:text "${text}")} } .\n` + : "" + }}`).then((json) => parseInt(json.results.bindings[0].count.value, 10)); } export async function getBasicMeta(id: string) { - return await fetch("http://fuseki:3030/3DOC/query", { - method: "POST", - headers: { - "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-query", - }, - body: "PREFIX rdf: \n" + - "PREFIX s: \n" + - "SELECT ?title ?date\n" + - "WHERE {\n" + - ' ?s s:identifier "' + id + '" .\n' + - " ?s s:dateCreated ?date .\n" + - " OPTIONAL { ?s s:name ?title . }\n" + - "}", - }).then((response) => response.json()).then((json: SparqlJson) => { + return await fusekiFetch(` +PREFIX rdf: +PREFIX s: +SELECT ?title ?date +WHERE { + ?s s:identifier "${id}" . + ?s s:dateCreated ?date . + OPTIONAL { ?s s:name ?title . } +}`).then((json) => { return { title: json.results.bindings[0]?.title?.value, created: json.results.bindings[0]?.date?.value, @@ -230,21 +188,14 @@ export async function getBasicMeta(id: string) { }); } -export async function getTagTypes(labels: string[]): Promise { - const response = await fetch("http://fuseki:3030/3DOC/query", { - method: "POST", - headers: { - "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-query", - }, - body: `PREFIX tridoc: +export async function getTagTypes(labels: string[]) { + const json = await fusekiFetch(` +PREFIX tridoc: SELECT DISTINCT ?l ?t WHERE { VALUES ?l { "${ - labels.join('" "') - }" } ?s tridoc:label ?l . OPTIONAL { ?s tridoc:valueType ?t . } }`, - }); - const json = await response.json(); + labels.join('" "') + }" } ?s tridoc:label ?l . OPTIONAL { ?s tridoc:valueType ?t . } }`); return json.results.bindings.map( - (binding: Record) => { + (binding) => { const result_1 = []; result_1[0] = binding.l.value; if (binding.t) { diff --git a/src/meta/fusekiFetch.ts b/src/meta/fusekiFetch.ts index bc607a8..84b69d7 100644 --- a/src/meta/fusekiFetch.ts +++ b/src/meta/fusekiFetch.ts @@ -8,7 +8,7 @@ type SparqlJson = { }; export async function fusekiFetch(query: string): Promise { - console.log((new Date()).toISOString(), "→ FUSEKI", query); + console.log((new Date()).toISOString(), "→ FUSEKI QUERY", query, "\n"); return await fetch("http://fuseki:3030/3DOC/query", { method: "POST", headers: { @@ -16,11 +16,27 @@ export async function fusekiFetch(query: string): Promise { "Content-Type": "application/sparql-query", }, body: query, - }).then((response) => { + }).then(async (response) => { if (response.ok) { return response.json(); } else { - throw new Error("Fuseki Error: " + response); + throw new Error("Fuseki Error: " + await response.text()); + } + }); +} + +export async function fusekiUpdate(query: string): Promise { + console.log((new Date()).toISOString(), "→ FUSEKI UPDATE", query, "\n"); + return await fetch("http://fuseki:3030/3DOC/update", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", + }, + body: query, + }).then(async (response) => { + if (!response.ok) { + throw new Error("Fuseki Error: " + await response.text()); } }); } diff --git a/src/meta/store.ts b/src/meta/store.ts index 9b23424..d7116ca 100644 --- a/src/meta/store.ts +++ b/src/meta/store.ts @@ -1,3 +1,5 @@ +import { fusekiUpdate } from "./fusekiFetch.ts"; + function escapeLiteral(string: string) { return string.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace( /\r/g, @@ -6,18 +8,11 @@ function escapeLiteral(string: string) { } export function restore(turtleData: string) { - const statement = `CLEAR GRAPH ; - INSERT DATA { - GRAPH { ${turtleData} } - }`; - return fetch("http://fuseki:3030/3DOC/update", { - method: "POST", - headers: { - "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-update", - }, - body: statement, - }); + return fusekiUpdate(` +CLEAR GRAPH ; +INSERT DATA { + GRAPH { ${turtleData} } +}`); } export async function storeDocument( @@ -36,19 +31,5 @@ INSERT DATA { s:text "${escapeLiteral(text)}" . } }`; - return await fetch("http://fuseki:3030/3DOC/update", { - method: "POST", - headers: { - "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-update", - }, - body: query, - }).then((response) => { - //console.log("Fuseki returned: "+response.status); - if (response.ok) { - return response; - } else { - throw new Error("Error from Fuseki: " + response.statusText); - } - }); + return await fusekiUpdate(query); } From 1d4159873fbd76d306881e3d24e053e3a09d5b8c Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 12:02:18 +0000 Subject: [PATCH 18/35] added POST /doc/:id/comment --- src/handlers/doc.ts | 9 +++++++++ src/meta/store.ts | 19 +++++++++++++++++++ src/server/routes.ts | 2 +- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 9319fa6..919bfb6 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -79,6 +79,15 @@ export async function list( return respond(JSON.stringify(response)); } +export async function postComment( + request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + await metastore.addComment(id, await request.text()); + return respond(undefined, { status: 204 }); +} + export async function postPDF( request: Request, _match: URLPatternResult, diff --git a/src/meta/store.ts b/src/meta/store.ts index d7116ca..2a590c5 100644 --- a/src/meta/store.ts +++ b/src/meta/store.ts @@ -7,6 +7,25 @@ function escapeLiteral(string: string) { ).replace(/'/g, "\\'").replace(/"/g, '\\"'); } +export async function addComment(id: string, text: string) { + const now = new Date(); + const query = ` +PREFIX rdf: +PREFIX xsd: +PREFIX tridoc: +PREFIX s: +INSERT DATA { + GRAPH { + s:comment [ + a s:Comment ; + s:dateCreated "${now.toISOString()}"^^xsd:dateTime ; + s:text "${escapeLiteral(text)}" + ] . + } +}`; + return await fusekiUpdate(query); +} + export function restore(turtleData: string) { return fusekiUpdate(` CLEAR GRAPH ; diff --git a/src/server/routes.ts b/src/server/routes.ts index 5fed9e9..32149d4 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -63,7 +63,7 @@ export const routes: { handler: doc.postPDF, }, { pattern: new URLPattern({ pathname: "/doc/:id/comment" }), - handler: notImplemented, + handler: doc.postComment, }, { pattern: new URLPattern({ pathname: "/doc/:id/tag" }), handler: notImplemented, From a9828f30de8f54a42c9443896147a6a80c8ce33c Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 12:22:08 +0000 Subject: [PATCH 19/35] added GET /doc/:id/thumb --- src/handlers/doc.ts | 52 ++++++++++++++++++++++++++++++++++++++++- src/handlers/raw.ts | 4 ++-- src/meta/fusekiFetch.ts | 2 +- src/server/routes.ts | 2 +- src/server/server.ts | 6 ++--- 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 919bfb6..d5ba44d 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -63,13 +63,63 @@ export async function getPDF( }, }); } catch (error) { - if (!(error instanceof Deno.errors.NotFound)) { + if (error instanceof Deno.errors.NotFound) { return respond("404 Not Found", { status: 404 }); } throw error; } } +export async function getThumb( + _request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + const path = getPath(id); + const fileName = await metafinder.getBasicMeta(id).then(( + { title, created }, + ) => title || created || "document"); + let thumb: Deno.FsFile; + try { + thumb = await Deno.open(path + ".png", { read: true }); + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + try { + await Deno.stat(path); // Check if PDF exists → 404 otherwise + const p = Deno.run({ + cmd: [ + "convert", + "-thumbnail", + "300x", + "-alpha", + "remove", + `${path}[0]`, + `${path}.png`, + ], + }); + const { success, code } = await p.status(); + if (!success) throw new Error("convert failed with code " + code); + thumb = await Deno.open(path + ".png", { read: true }); + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + return respond("404 Not Found", { status: 404 }); + } + throw error; + } + } else { + throw error; + } + } + // Build a readable stream so the file doesn't have to be fully loaded into memory while we send it + const readableStream = thumb.readable; + return respond(readableStream, { + headers: { + "content-disposition": `inline; filename="${encodeURI(fileName)}.png"`, + "content-type": "image/png", + }, + }); +} + export async function list( request: Request, _match: URLPatternResult, diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts index c819489..22ab9f5 100644 --- a/src/handlers/raw.ts +++ b/src/handlers/raw.ts @@ -9,7 +9,7 @@ export async function deleteRdfFile( _match: URLPatternResult, ): Promise { await Deno.remove("rdf.ttl"); - return respond("200: OK"); + return respond(undefined, { status: 204 }); } export async function putZip( @@ -38,5 +38,5 @@ export async function putZip( const turtleData = decoder.decode(await Deno.readFile("rdf.ttl")); await Deno.remove("rdf.ttl"); await restore(turtleData); - return respond("200: OK"); + return respond(undefined, { status: 204 }); } diff --git a/src/meta/fusekiFetch.ts b/src/meta/fusekiFetch.ts index 84b69d7..f28f551 100644 --- a/src/meta/fusekiFetch.ts +++ b/src/meta/fusekiFetch.ts @@ -31,7 +31,7 @@ export async function fusekiUpdate(query: string): Promise { method: "POST", headers: { "Authorization": "Basic " + btoa("admin:pw123"), - "Content-Type": "application/sparql-query", + "Content-Type": "application/sparql-update", }, body: query, }).then(async (response) => { diff --git a/src/server/routes.ts b/src/server/routes.ts index 32149d4..7e06bff 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -32,7 +32,7 @@ export const routes: { handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/doc/:id/thumb" }), - handler: notImplemented, + handler: doc.getThumb, }, { pattern: new URLPattern({ pathname: "/doc/:id/title" }), handler: notImplemented, diff --git a/src/server/server.ts b/src/server/server.ts index d49f105..4479ff1 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -19,7 +19,7 @@ const handler = async (request: Request): Promise => { path, "→ 401: Not Authenticated", ); - return respond("401: Not Authenticated", { + return respond("401 Not Authenticated", { status: 401, headers: { "WWW-Authenticate": "Basic" }, }); @@ -38,7 +38,7 @@ const handler = async (request: Request): Promise => { path, "→ 404: Path not found", ); - return respond("404: Path not found", { status: 404 }); + return respond("404 Path not found", { status: 404 }); } catch (error) { console.log( (new Date()).toISOString(), @@ -47,7 +47,7 @@ const handler = async (request: Request): Promise => { "→ 500: ", error, ); - return respond("500: " + error, { status: 500 }); + return respond("500 " + error, { status: 500 }); } }; From 6b67780b58f2fdef424d4c507c2026c6d8fb6ffa Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 12:32:24 +0000 Subject: [PATCH 20/35] added GET /doc/:id/title --- src/handlers/doc.ts | 10 +++++++++- src/server/routes.ts | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index d5ba44d..50ae584 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -78,7 +78,7 @@ export async function getThumb( const path = getPath(id); const fileName = await metafinder.getBasicMeta(id).then(( { title, created }, - ) => title || created || "document"); + ) => title || created || "thumbnail"); let thumb: Deno.FsFile; try { thumb = await Deno.open(path + ".png", { read: true }); @@ -120,6 +120,14 @@ export async function getThumb( }); } +export async function getTitle( + _request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + return respond((await metafinder.getBasicMeta(id)).title); +} + export async function list( request: Request, _match: URLPatternResult, diff --git a/src/server/routes.ts b/src/server/routes.ts index 7e06bff..63fc577 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -35,7 +35,7 @@ export const routes: { handler: doc.getThumb, }, { pattern: new URLPattern({ pathname: "/doc/:id/title" }), - handler: notImplemented, + handler: doc.getTitle, }, { pattern: new URLPattern({ pathname: "/doc/:id/meta" }), handler: notImplemented, From c5576efc5e398735fb241ded323175178642e6aa Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 12:47:33 +0000 Subject: [PATCH 21/35] added GET /doc/:id/tag, /doc/:id/meta --- README.md | 10 +++++----- src/handlers/doc.ts | 24 +++++++++++++++++++++++- src/meta/finder.ts | 34 ++++++++++++++++++++++++++++++++++ src/server/routes.ts | 4 ++-- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7664b31..16d8a3b 100644 --- a/README.md +++ b/README.md @@ -112,16 +112,16 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | | `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | ✅ | | `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | ✅ | -| `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | +| `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | ✅ | | `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | ✅ | | `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | -| `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | +| `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | ✅ | | `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 | -| `/doc/{id}/thumb` | GET | Get document thumbnail | - | PNG (300px wide) | 1.5.0 | +| `/doc/{id}/thumb` | GET | Get document thumbnail | - | PNG (300px wide) | 1.5.0 | ✅ | | `/doc/{id}/title` | PUT | Set document title | `{"title": "the_Title"}` | - | 1.1.0 | -| `/doc/{id}/title` | GET | Get document title | - | `{"title": "the_Title"}` | 1.1.0 | +| `/doc/{id}/title` | GET | Get document title | - | `{"title": "the_Title"}` | 1.1.0 | ✅ | | `/doc/{id}/title` | DELETE | Reset document title | - | - | 1.1.0 | -| `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | +| `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | ✅ | | `/raw/rdf` | GET | Get all metadata as RDF. Useful for Backups | [4](#f4) | RDF, Content-Type defined over request Headers or ?accept. Fallback to text/turtle. | 1.1.0 | | `/raw/rdf` | DELETE | "Cancel" failed zip upload—use only if certain it’s done & failed | | | (deno only) | ✅ | | `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 50ae584..b82650c 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -70,6 +70,28 @@ export async function getPDF( } } +export async function getMeta( + _request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + return respond( + JSON.stringify({ + ...(await metafinder.getBasicMeta(id)), + comments: await metafinder.getComments(id), + tags: await metafinder.getTags(id), + }), + ); +} + +export async function getTags( + _request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + return respond(JSON.stringify(await metafinder.getTags(id))); +} + export async function getThumb( _request: Request, match: URLPatternResult, @@ -142,7 +164,7 @@ export async function postComment( match: URLPatternResult, ): Promise { const id = match.pathname.groups.id; - await metastore.addComment(id, await request.text()); + await metastore.addComment(id, (await request.json()).text); return respond(undefined, { status: 204 }); } diff --git a/src/meta/finder.ts b/src/meta/finder.ts index 15bdf63..f894d35 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -188,6 +188,40 @@ WHERE { }); } +export async function getTags(id: string) { + const query = ` +PREFIX rdf: +PREFIX xsd: +PREFIX tridoc: +PREFIX s: +SELECT DISTINCT ?label ?type ?v + WHERE { + GRAPH { + tridoc:tag ?tag . + { + ?tag tridoc:label ?label . + } + UNION + { + ?tag tridoc:value ?v ; + tridoc:parameterizableTag ?ptag . + ?ptag tridoc:label ?label ; + tridoc:valueType ?type . + } + } +}`; + return await fusekiFetch(query).then((json) => + json.results.bindings.map((binding) => { + return { + label: binding.label.value, + parameter: binding.type + ? { type: binding.type.value, value: binding.v.value } + : undefined, + }; + }) + ); +} + export async function getTagTypes(labels: string[]) { const json = await fusekiFetch(` PREFIX tridoc: diff --git a/src/server/routes.ts b/src/server/routes.ts index 63fc577..f4435d6 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -29,7 +29,7 @@ export const routes: { handler: doc.getComments, }, { pattern: new URLPattern({ pathname: "/doc/:id/tag" }), - handler: notImplemented, + handler: doc.getTags, }, { pattern: new URLPattern({ pathname: "/doc/:id/thumb" }), handler: doc.getThumb, @@ -38,7 +38,7 @@ export const routes: { handler: doc.getTitle, }, { pattern: new URLPattern({ pathname: "/doc/:id/meta" }), - handler: notImplemented, + handler: doc.getMeta, }, { pattern: new URLPattern({ pathname: "/raw/rdf" }), handler: notImplemented, From 4187119303be81b321e1bbe4c4e5c3b06b76eaed Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 13:07:15 +0000 Subject: [PATCH 22/35] added GET /raw/rdf --- README.md | 2 +- src/handlers/raw.ts | 12 ++++++++++++ src/meta/fusekiFetch.ts | 14 ++++++++++++++ src/server/routes.ts | 2 +- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16d8a3b..7942afe 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc/{id}/title` | GET | Get document title | - | `{"title": "the_Title"}` | 1.1.0 | ✅ | | `/doc/{id}/title` | DELETE | Reset document title | - | - | 1.1.0 | | `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | ✅ | -| `/raw/rdf` | GET | Get all metadata as RDF. Useful for Backups | [4](#f4) | RDF, Content-Type defined over request Headers or ?accept. Fallback to text/turtle. | 1.1.0 | +| `/raw/rdf` | GET | Get all metadata as RDF. Useful for Backups | [4](#f4) | RDF, Content-Type defined over request Headers or ?accept. Fallback to text/turtle. | 1.1.0 | ✅ | | `/raw/rdf` | DELETE | "Cancel" failed zip upload—use only if certain it’s done & failed | | | (deno only) | ✅ | | `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | | `/raw/zip` | PUT | Replace all data with backup zip | ZIP | Replaces the metadata and adds the blobs from the zip | 1.3.0 | ✅ | diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts index 22ab9f5..e63a12c 100644 --- a/src/handlers/raw.ts +++ b/src/handlers/raw.ts @@ -1,5 +1,6 @@ import { emptyDir, writableStreamFromWriter } from "../deps.ts"; import { respond } from "../helpers/cors.ts"; +import { dump } from "../meta/fusekiFetch.ts"; import { restore } from "../meta/store.ts"; const decoder = new TextDecoder("utf-8"); @@ -12,6 +13,17 @@ export async function deleteRdfFile( return respond(undefined, { status: 204 }); } +export async function getRdf( + request: Request, + _match: URLPatternResult, +): Promise { + const url = new URL(request.url); + const accept = url.searchParams.has("accept") + ? decodeURIComponent(url.searchParams.get("accept")!) + : request.headers.get("Accept") || "text/turtle"; + return await dump(accept); +} + export async function putZip( request: Request, _match: URLPatternResult, diff --git a/src/meta/fusekiFetch.ts b/src/meta/fusekiFetch.ts index f28f551..afbaf5a 100644 --- a/src/meta/fusekiFetch.ts +++ b/src/meta/fusekiFetch.ts @@ -7,6 +7,20 @@ type SparqlJson = { }; }; +export function dump(accept = "text/turtle") { + const query = "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }"; + console.log((new Date()).toISOString(), "→ FUSEKI QUERY", query, "\n"); + return fetch("http://fuseki:3030/3DOC/query", { + method: "POST", + headers: { + "Authorization": "Basic " + btoa("admin:pw123"), + "Content-Type": "application/sparql-query", + "Accept": accept, + }, + body: query, + }); +} + export async function fusekiFetch(query: string): Promise { console.log((new Date()).toISOString(), "→ FUSEKI QUERY", query, "\n"); return await fetch("http://fuseki:3030/3DOC/query", { diff --git a/src/server/routes.ts b/src/server/routes.ts index f4435d6..5fb3cf9 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -41,7 +41,7 @@ export const routes: { handler: doc.getMeta, }, { pattern: new URLPattern({ pathname: "/raw/rdf" }), - handler: notImplemented, + handler: raw.getRdf, }, { pattern: new URLPattern({ pathname: "/raw/zip" }), handler: notImplemented, From 520a4d006cab15fbd66224cbe5eac877229cc7ee Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 14:29:26 +0000 Subject: [PATCH 23/35] added GET /raw/tgz --- .devcontainer/docker-cmd.sh | 2 +- deno.jsonc | 5 +- docker-cmd.sh | 2 +- src/handlers/raw.ts | 105 ++++++++++++++++++++++++++++++++++-- src/server/routes.ts | 8 +-- 5 files changed, 111 insertions(+), 11 deletions(-) diff --git a/.devcontainer/docker-cmd.sh b/.devcontainer/docker-cmd.sh index cfe9532..6791b5b 100644 --- a/.devcontainer/docker-cmd.sh +++ b/.devcontainer/docker-cmd.sh @@ -3,7 +3,7 @@ echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' set -m -deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts & +deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts & sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ diff --git a/deno.jsonc b/deno.jsonc index 5413c78..03da2df 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -5,7 +5,8 @@ } }, "tasks": { - "run": "deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttls --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts", - "run-watch": "deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts" + // --allow-run=convert,pdfsandwich,pdftotext,tar,zip,unzip,bash + "run": "deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttls --allow-run --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts", + "run-watch": "deno run --watch --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts" } } diff --git a/docker-cmd.sh b/docker-cmd.sh index 0780872..c707f9c 100644 --- a/docker-cmd.sh +++ b/docker-cmd.sh @@ -3,7 +3,7 @@ echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' --data 'dbName=3DOC&dbType=tdb' set -m -deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run=convert,pdfsandwich,pdftotext,zip,unzip --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts & +deno run --allow-net --allow-read=blobs,rdf.ttl --allow-write=blobs,rdf.ttl --allow-run --allow-env=TRIDOC_PWD,OCR_LANG src/main.ts & sleep 5 echo 'Attempting to create Dataset "3DOC"' curl 'http://fuseki:3030/$/datasets' -H "Authorization: Basic $(echo -n admin:pw123 | base64)" \ diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts index e63a12c..b233700 100644 --- a/src/handlers/raw.ts +++ b/src/handlers/raw.ts @@ -1,3 +1,4 @@ +import { ensureDir } from "https://deno.land/std@0.160.0/fs/ensure_dir.ts"; import { emptyDir, writableStreamFromWriter } from "../deps.ts"; import { respond } from "../helpers/cors.ts"; import { dump } from "../meta/fusekiFetch.ts"; @@ -5,7 +6,7 @@ import { restore } from "../meta/store.ts"; const decoder = new TextDecoder("utf-8"); -export async function deleteRdfFile( +export async function deleteRDFFile( _request: Request, _match: URLPatternResult, ): Promise { @@ -13,7 +14,7 @@ export async function deleteRdfFile( return respond(undefined, { status: 204 }); } -export async function getRdf( +export async function getRDF( request: Request, _match: URLPatternResult, ): Promise { @@ -24,7 +25,105 @@ export async function getRdf( return await dump(accept); } -export async function putZip( +export async function getTGZ( + _request: Request, + _match: URLPatternResult, +): Promise { + const timestamp = "" + Date.now(); + const tarPath = "blobs/tgz-" + timestamp; + const rdfName = "rdf-" + timestamp; + const rdfPath = "blobs/rdf/" + rdfName; + await ensureDir("blobs/rdf"); + const rdf = await Deno.open(rdfPath, { + create: true, + write: true, + truncate: true, + }); + const writableStream = writableStreamFromWriter(rdf); + await (await dump()).body?.pipeTo(writableStream); + /* const p = Deno.run({ + cmd: [ + "tar", + `--transform=s|${rdfPath}|rdf.ttl|`, + `--exclude-tag=${rdfName}`, + "-czvf", + tarPath, + "blobs/* /", // no space! + ], + }); */ + const p = Deno.run({ + cmd: [ + "bash", + "-c", + `tar --transform="s|${rdfPath}|rdf.ttl|" --exclude-tag="${rdfName}" -czvf ${tarPath} blobs/*/`, + ], + }); + const { success, code } = await p.status(); + if (!success) throw new Error("tar -czf failed with code " + code); + await Deno.remove(rdfPath); + const tar = await Deno.open(tarPath); + // Build a readable stream so the file doesn't have to be fully loaded into memory while we send it + const readableStream = tar.readable; + return respond(readableStream, { + headers: { + "content-disposition": + `inline; filename="tridoc_backup_${timestamp}.tar.gz"`, + "content-type": "application/gzip", + }, + }); + // TODO: Figure out how to delete these files +} + +export async function getZIP( + _request: Request, + _match: URLPatternResult, +): Promise { + const timestamp = "" + Date.now(); + const tarPath = "blobs/tgz-" + timestamp; + const rdfName = "rdf-" + timestamp; + const rdfPath = "blobs/rdf/" + rdfName; + await ensureDir("blobs/rdf"); + const rdf = await Deno.open(rdfPath, { + create: true, + write: true, + truncate: true, + }); + const writableStream = writableStreamFromWriter(rdf); + await (await dump()).body?.pipeTo(writableStream); + /* const p = Deno.run({ + cmd: [ + "tar", + `--transform=s|${rdfPath}|rdf.ttl|`, + `--exclude-tag=${rdfName}`, + "-czvf", + tarPath, + "blobs/* /", // no space! + ], + }); */ + const p = Deno.run({ + cmd: [ + "bash", + "-c", + `tar --transform="s|${rdfPath}|rdf.ttl|" --exclude-tag="${rdfName}" -czvf ${tarPath} blobs/*/`, + ], + }); + const { success, code } = await p.status(); + if (!success) throw new Error("tar -czf failed with code " + code); + await Deno.remove(rdfPath); + const tar = await Deno.open(tarPath); + // Build a readable stream so the file doesn't have to be fully loaded into memory while we send it + const readableStream = tar.readable; + return respond(readableStream, { + headers: { + "content-disposition": + `inline; filename="tridoc_backup_${timestamp}.tar.gz"`, + "content-type": "application/gzip", + }, + }); + // TODO: Figure out how to delete these files +} + +export async function putZIP( request: Request, _match: URLPatternResult, ): Promise { diff --git a/src/server/routes.ts b/src/server/routes.ts index 5fb3cf9..dc1940b 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -41,13 +41,13 @@ export const routes: { handler: doc.getMeta, }, { pattern: new URLPattern({ pathname: "/raw/rdf" }), - handler: raw.getRdf, + handler: raw.getRDF, }, { pattern: new URLPattern({ pathname: "/raw/zip" }), handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/raw/tgz" }), - handler: notImplemented, + handler: raw.getTGZ, }, { pattern: new URLPattern({ pathname: "/tag" }), handler: notImplemented, @@ -76,7 +76,7 @@ export const routes: { handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/raw/zip" }), - handler: raw.putZip, + handler: raw.putZIP, }], "DELETE": [{ pattern: new URLPattern({ pathname: "/doc/:id" }), @@ -92,6 +92,6 @@ export const routes: { handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/raw/rdf" }), - handler: raw.deleteRdfFile, + handler: raw.deleteRDFFile, }], }; From 8151fe3e0f30bd01524a09cf6789e05725c24e1d Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 15:17:19 +0000 Subject: [PATCH 24/35] added GET /raw/zip --- src/handlers/raw.ts | 49 +++++++++++++++++--------------------------- src/server/routes.ts | 2 +- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/handlers/raw.ts b/src/handlers/raw.ts index b233700..a680a83 100644 --- a/src/handlers/raw.ts +++ b/src/handlers/raw.ts @@ -41,16 +41,6 @@ export async function getTGZ( }); const writableStream = writableStreamFromWriter(rdf); await (await dump()).body?.pipeTo(writableStream); - /* const p = Deno.run({ - cmd: [ - "tar", - `--transform=s|${rdfPath}|rdf.ttl|`, - `--exclude-tag=${rdfName}`, - "-czvf", - tarPath, - "blobs/* /", // no space! - ], - }); */ const p = Deno.run({ cmd: [ "bash", @@ -79,10 +69,8 @@ export async function getZIP( _match: URLPatternResult, ): Promise { const timestamp = "" + Date.now(); - const tarPath = "blobs/tgz-" + timestamp; - const rdfName = "rdf-" + timestamp; - const rdfPath = "blobs/rdf/" + rdfName; - await ensureDir("blobs/rdf"); + const zipPath = `blobs/zip-${timestamp}.zip`; + const rdfPath = "blobs/rdf-" + timestamp; const rdf = await Deno.open(rdfPath, { create: true, write: true, @@ -90,34 +78,35 @@ export async function getZIP( }); const writableStream = writableStreamFromWriter(rdf); await (await dump()).body?.pipeTo(writableStream); - /* const p = Deno.run({ + // Create zip + const p_1 = Deno.run({ cmd: [ - "tar", - `--transform=s|${rdfPath}|rdf.ttl|`, - `--exclude-tag=${rdfName}`, - "-czvf", - tarPath, - "blobs/* /", // no space! + "bash", + "-c", + `zip -r ${zipPath} blobs/*/ ${rdfPath} -x "blobs/rdf/*"`, ], - }); */ - const p = Deno.run({ + }); + const r_1 = await p_1.status(); + if (!r_1.success) throw new Error("zip failed with code " + r_1.code); + // move rdf-??? to rdf.zip + const p_2 = Deno.run({ cmd: [ "bash", "-c", - `tar --transform="s|${rdfPath}|rdf.ttl|" --exclude-tag="${rdfName}" -czvf ${tarPath} blobs/*/`, + `printf "@ ${rdfPath}\\n@=rdf.ttl\\n" | zipnote -w ${zipPath}`, ], }); - const { success, code } = await p.status(); - if (!success) throw new Error("tar -czf failed with code " + code); + const r_2 = await p_2.status(); + if (!r_2.success) throw new Error("zipnote failed with code " + r_2.code); await Deno.remove(rdfPath); - const tar = await Deno.open(tarPath); + const zip = await Deno.open(zipPath); // Build a readable stream so the file doesn't have to be fully loaded into memory while we send it - const readableStream = tar.readable; + const readableStream = zip.readable; return respond(readableStream, { headers: { "content-disposition": - `inline; filename="tridoc_backup_${timestamp}.tar.gz"`, - "content-type": "application/gzip", + `inline; filename="tridoc_backup_${timestamp}.zip"`, + "content-type": "application/zip", }, }); // TODO: Figure out how to delete these files diff --git a/src/server/routes.ts b/src/server/routes.ts index dc1940b..1898af9 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -44,7 +44,7 @@ export const routes: { handler: raw.getRDF, }, { pattern: new URLPattern({ pathname: "/raw/zip" }), - handler: notImplemented, + handler: raw.getZIP, }, { pattern: new URLPattern({ pathname: "/raw/tgz" }), handler: raw.getTGZ, From 6050e11ef2e57e997cdd13c894923b76b8a2bab7 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 15:55:52 +0000 Subject: [PATCH 25/35] rm broken --- .vscode/launch.json | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 01e694c..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "node", - "request": "launch", - "name": "Launch Program", - "program": "${workspaceFolder}/lib/server", - "env": {"TRIDOC_PWD": "tridoc"} - } - ] -} \ No newline at end of file From 93bfaa63b1d3a8038803c50f14c9509d6dbc1fd0 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 17:11:18 +0000 Subject: [PATCH 26/35] added GET /tag/:tagLabel --- README.md | 6 ++--- src/handlers/tag.ts | 39 ++++++++++++++++++++++++++++ src/helpers/processParams.ts | 12 ++++++++- src/meta/finder.ts | 49 +++++++++++++++++++++++------------- src/server/routes.ts | 5 ++-- 5 files changed, 88 insertions(+), 23 deletions(-) create mode 100644 src/handlers/tag.ts diff --git a/README.md b/README.md index 7942afe..b966510 100644 --- a/README.md +++ b/README.md @@ -124,11 +124,11 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | ✅ | | `/raw/rdf` | GET | Get all metadata as RDF. Useful for Backups | [4](#f4) | RDF, Content-Type defined over request Headers or ?accept. Fallback to text/turtle. | 1.1.0 | ✅ | | `/raw/rdf` | DELETE | "Cancel" failed zip upload—use only if certain it’s done & failed | | | (deno only) | ✅ | -| `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | +| `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | ✅ | | `/raw/zip` | PUT | Replace all data with backup zip | ZIP | Replaces the metadata and adds the blobs from the zip | 1.3.0 | ✅ | | `/tag` | POST | Create new tag | See above | - | 1.1.0 | -| `/tag` | GET | Get (list of) all tags | - | - | 1.1.0 | -| `/tag/{tagLabel}` | GET | Get Documents with this tag. Same as `/doc?tag={tagLabel}` | [1](#f1) [2](#f2) | Array of objects with document identifiers and titles (where available) | 1.1.0 | +| `/tag` | GET | Get (list of) all tags | - | - | 1.1.0 | ✅ | +| `/tag/{tagLabel}` | GET | Get Documents with this tag. Same as `/doc?tag={tagLabel}` | [1](#f1) [2](#f2) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | | `/tag/{tagLabel}` | DELETE | Delete this tag | - | - | 1.1.0 | | `/version` | GET | Get tridoc version | - | semver version number | 1.1.0 | ✅ | diff --git a/src/handlers/tag.ts b/src/handlers/tag.ts new file mode 100644 index 0000000..7d5ed81 --- /dev/null +++ b/src/handlers/tag.ts @@ -0,0 +1,39 @@ +import { respond } from "../helpers/cors.ts"; +import { processParams } from "../helpers/processParams.ts"; +import * as metafinder from "../meta/finder.ts"; + +type TagCreate = { + label: string; + parameter?: { + type: + | "http://www.w3.org/2001/XMLSchema#decimal" + | "http://www.w3.org/2001/XMLSchema#date"; + }; // only for parameterizable tags +}; +type TagAdd = { + label: string; + parameter?: { + type: + | "http://www.w3.org/2001/XMLSchema#decimal" + | "http://www.w3.org/2001/XMLSchema#date"; + value: string; // must be valid xsd:decimal or xsd:date, as specified in property type. + }; // only for parameterizable tags +}; + +export async function getDocs( + request: Request, + match: URLPatternResult, +): Promise { + const params = await processParams(request, { + tags: [[match.pathname.groups.tagLabel]], + }); + const response = await metafinder.getDocumentList(params); + return respond(JSON.stringify(response)); +} + +export async function getTagList( + _request: Request, + _match: URLPatternResult, +): Promise { + return respond(JSON.stringify(await metafinder.getTagList())); +} diff --git a/src/helpers/processParams.ts b/src/helpers/processParams.ts index d02de40..d03525a 100644 --- a/src/helpers/processParams.ts +++ b/src/helpers/processParams.ts @@ -19,6 +19,11 @@ type ParamTag = { maxIsExclusive?: boolean; //[5] }; +export type queryOverrides = { + tags?: string[][]; + nottags?: string[][]; +}; + export type Params = { tags?: ParamTag[]; nottags?: ParamTag[]; @@ -27,11 +32,16 @@ export type Params = { offset?: number; }; -export async function processParams(request: Request): Promise { +export async function processParams( + request: Request, + queryOverrides?: queryOverrides, +): Promise { const query = extractQuery(request); const result: Params = {}; const tags = query.tag?.map((t) => t.split(";")) ?? []; + if (queryOverrides?.tags) tags.push(...queryOverrides.tags); const nottags = query.nottag?.map((t) => t.split(";")) ?? []; + if (queryOverrides?.nottags) tags.push(...queryOverrides.nottags); result.text = query.text?.[0]; result.limit = parseInt(query.limit?.[0], 10) > 0 ? parseInt(query.limit[0]) diff --git a/src/meta/finder.ts b/src/meta/finder.ts index f894d35..06fb9b1 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -188,26 +188,41 @@ WHERE { }); } +export async function getTagList() { + const query = ` +PREFIX tridoc: +SELECT DISTINCT ?s ?label ?type +WHERE { + ?s tridoc:label ?label . + OPTIONAL { ?s tridoc:valueType ?type . } +}`; + return await fusekiFetch(query).then((json) => + json.results.bindings.map((binding) => { + return { + label: binding.label.value, + parameter: binding.type ? { type: binding.type.value } : undefined, + }; + }) + ); +} + export async function getTags(id: string) { const query = ` -PREFIX rdf: -PREFIX xsd: -PREFIX tridoc: -PREFIX s: -SELECT DISTINCT ?label ?type ?v - WHERE { - GRAPH { - tridoc:tag ?tag . +PREFIX tridoc: +SELECT DISTINCT ?label ?type ?v + WHERE { + GRAPH { + tridoc:tag ?tag . { - ?tag tridoc:label ?label . - } - UNION - { - ?tag tridoc:value ?v ; - tridoc:parameterizableTag ?ptag . - ?ptag tridoc:label ?label ; - tridoc:valueType ?type . - } + ?tag tridoc:label ?label . + } + UNION + { + ?tag tridoc:value ?v ; + tridoc:parameterizableTag ?ptag . + ?ptag tridoc:label ?label ; + tridoc:valueType ?type . + } } }`; return await fusekiFetch(query).then((json) => diff --git a/src/server/routes.ts b/src/server/routes.ts index 1898af9..3ec9967 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -3,6 +3,7 @@ import { count } from "../handlers/count.ts"; import * as doc from "../handlers/doc.ts"; import { notImplemented } from "../handlers/notImplemented.ts"; import * as raw from "../handlers/raw.ts"; +import * as tag from "../handlers/tag.ts"; import { version } from "../handlers/version.ts"; export const routes: { @@ -50,10 +51,10 @@ export const routes: { handler: raw.getTGZ, }, { pattern: new URLPattern({ pathname: "/tag" }), - handler: notImplemented, + handler: tag.getTagList, }, { pattern: new URLPattern({ pathname: "/tag/:tagLabel" }), - handler: notImplemented, + handler: tag.getDocs, }, { pattern: new URLPattern({ pathname: "/version" }), handler: version, From b0384de2a9ef7621938eea66322a51da63fcf5e3 Mon Sep 17 00:00:00 2001 From: nleanba Date: Sun, 23 Oct 2022 17:41:48 +0000 Subject: [PATCH 27/35] added POST /doc/:id/tag --- README.md | 2 +- src/handlers/doc.ts | 34 ++++++++++++++++++++++++++++++++++ src/handlers/tag.ts | 9 --------- src/meta/finder.ts | 1 + src/meta/store.ts | 29 +++++++++++++++++++++++++++++ src/server/routes.ts | 2 +- 6 files changed, 66 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b966510..fa419c0 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | ✅ | | `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | ✅ | | `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | ✅ | -| `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | +| `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | ✅ | | `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | ✅ | | `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 | | `/doc/{id}/thumb` | GET | Get document thumbnail | - | PNG (300px wide) | 1.5.0 | ✅ | diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index b82650c..a0537f8 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -7,6 +7,16 @@ import * as metadelete from "../meta/delete.ts"; import * as metafinder from "../meta/finder.ts"; import * as metastore from "../meta/store.ts"; +type TagAdd = { + label: string; + parameter?: { + type: + | "http://www.w3.org/2001/XMLSchema#decimal" + | "http://www.w3.org/2001/XMLSchema#date"; + value: string; // must be valid xsd:decimal or xsd:date, as specified in property type. + }; // only for parameterizable tags +}; + function getDir(id: string) { return "./blobs/" + id.slice(0, 2) + "/" + id.slice(2, 6) + "/" + id.slice(6, 14); @@ -212,3 +222,27 @@ export async function postPDF( }, }); } + +export async function postTag( + request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + const tagObject: TagAdd = await request.json(); + const [label, type] = + (await metafinder.getTagTypes([tagObject.label]))?.[0] ?? + [undefined, undefined]; + if (!label) { + return respond("Tag must exist before adding to a document", { + status: 400, + }); + } + if (tagObject.parameter?.type !== type) { + return respond("Type provided does not match", { status: 400 }); + } + if (tagObject.parameter?.type && !tagObject.parameter?.value) { + return respond("No value provided", { status: 400 }); + } + await metastore.addTag(id, tagObject.label, tagObject.parameter?.value, type); + return respond(undefined, { status: 204 }); +} diff --git a/src/handlers/tag.ts b/src/handlers/tag.ts index 7d5ed81..562ac12 100644 --- a/src/handlers/tag.ts +++ b/src/handlers/tag.ts @@ -10,15 +10,6 @@ type TagCreate = { | "http://www.w3.org/2001/XMLSchema#date"; }; // only for parameterizable tags }; -type TagAdd = { - label: string; - parameter?: { - type: - | "http://www.w3.org/2001/XMLSchema#decimal" - | "http://www.w3.org/2001/XMLSchema#date"; - value: string; // must be valid xsd:decimal or xsd:date, as specified in property type. - }; // only for parameterizable tags -}; export async function getDocs( request: Request, diff --git a/src/meta/finder.ts b/src/meta/finder.ts index 06fb9b1..ad3ed14 100644 --- a/src/meta/finder.ts +++ b/src/meta/finder.ts @@ -237,6 +237,7 @@ SELECT DISTINCT ?label ?type ?v ); } +// => [label, type?][] export async function getTagTypes(labels: string[]) { const json = await fusekiFetch(` PREFIX tridoc: diff --git a/src/meta/store.ts b/src/meta/store.ts index 2a590c5..c4af271 100644 --- a/src/meta/store.ts +++ b/src/meta/store.ts @@ -26,6 +26,35 @@ INSERT DATA { return await fusekiUpdate(query); } +export async function addTag( + id: string, + label: string, + value: string, + type: string, +) { + const tag = value + ? encodeURIComponent(label) + "/" + value + : encodeURIComponent(label); + const query = ` +PREFIX rdf: +PREFIX xsd: +PREFIX tridoc: +PREFIX s: +INSERT DATA { + GRAPH { + tridoc:tag .${ + value + ? ` + a tridoc:ParameterizedTag ; + tridoc:parameterizableTag ; + tridoc:value "${value}"^^<${type}> .` + : "" + } + } +}`; + return await fusekiUpdate(query); +} + export function restore(turtleData: string) { return fusekiUpdate(` CLEAR GRAPH ; diff --git a/src/server/routes.ts b/src/server/routes.ts index 3ec9967..3b6edba 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -67,7 +67,7 @@ export const routes: { handler: doc.postComment, }, { pattern: new URLPattern({ pathname: "/doc/:id/tag" }), - handler: notImplemented, + handler: doc.postTag, }, { pattern: new URLPattern({ pathname: "/tag" }), handler: notImplemented, From 690595b0bafd3dbfd52854a33a65531dab19123d Mon Sep 17 00:00:00 2001 From: nleanba Date: Mon, 24 Oct 2022 17:45:26 +0000 Subject: [PATCH 28/35] added PUT /doc/:id/title and fixed GET /doc/:id/title --- README.md | 2 +- src/handlers/doc.ts | 14 +++++++++++++- src/meta/store.ts | 11 +++++++++++ src/server/routes.ts | 2 +- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fa419c0..1ab809f 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | ✅ | | `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 | | `/doc/{id}/thumb` | GET | Get document thumbnail | - | PNG (300px wide) | 1.5.0 | ✅ | -| `/doc/{id}/title` | PUT | Set document title | `{"title": "the_Title"}` | - | 1.1.0 | +| `/doc/{id}/title` | PUT | Set document title | `{"title": "the_Title"}` | - | 1.1.0 | ✅ | | `/doc/{id}/title` | GET | Get document title | - | `{"title": "the_Title"}` | 1.1.0 | ✅ | | `/doc/{id}/title` | DELETE | Reset document title | - | - | 1.1.0 | | `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | ✅ | diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index a0537f8..e359bc3 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -157,7 +157,9 @@ export async function getTitle( match: URLPatternResult, ): Promise { const id = match.pathname.groups.id; - return respond((await metafinder.getBasicMeta(id)).title); + return respond( + JSON.stringify({ title: (await metafinder.getBasicMeta(id)).title }), + ); } export async function list( @@ -246,3 +248,13 @@ export async function postTag( await metastore.addTag(id, tagObject.label, tagObject.parameter?.value, type); return respond(undefined, { status: 204 }); } + +export async function putTitle( + request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + const title: string = (await request.json())?.title; + await metastore.addTitle(id, title); + return respond(undefined, { status: 204 }); +} diff --git a/src/meta/store.ts b/src/meta/store.ts index c4af271..f2ac2ff 100644 --- a/src/meta/store.ts +++ b/src/meta/store.ts @@ -55,6 +55,17 @@ INSERT DATA { return await fusekiUpdate(query); } +export async function addTitle(id: string, title: string) { + const query = ` +PREFIX rdf: +PREFIX s: +WITH +DELETE { s:name ?o } +INSERT { s:name "${escapeLiteral(title)}" } +WHERE { OPTIONAL { s:name ?o } }`; + return await fusekiUpdate(query); +} + export function restore(turtleData: string) { return fusekiUpdate(` CLEAR GRAPH ; diff --git a/src/server/routes.ts b/src/server/routes.ts index 3b6edba..033a089 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -74,7 +74,7 @@ export const routes: { }], "PUT": [{ pattern: new URLPattern({ pathname: "/doc/:id/title" }), - handler: notImplemented, + handler: doc.putTitle, }, { pattern: new URLPattern({ pathname: "/raw/zip" }), handler: raw.putZIP, From 0479d8d6011b6847129da98e54777a3f9075cc2e Mon Sep 17 00:00:00 2001 From: nleanba Date: Mon, 24 Oct 2022 18:25:42 +0000 Subject: [PATCH 29/35] added POST /tag --- README.md | 2 +- src/handlers/tag.ts | 24 ++++++++++++++++++++++++ src/meta/store.ts | 22 ++++++++++++++++++++++ src/server/routes.ts | 2 +- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ab809f..e1ea603 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/raw/rdf` | DELETE | "Cancel" failed zip upload—use only if certain it’s done & failed | | | (deno only) | ✅ | | `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | ✅ | | `/raw/zip` | PUT | Replace all data with backup zip | ZIP | Replaces the metadata and adds the blobs from the zip | 1.3.0 | ✅ | -| `/tag` | POST | Create new tag | See above | - | 1.1.0 | +| `/tag` | POST | Create new tag | See above | - | 1.1.0 | ✅ | | `/tag` | GET | Get (list of) all tags | - | - | 1.1.0 | ✅ | | `/tag/{tagLabel}` | GET | Get Documents with this tag. Same as `/doc?tag={tagLabel}` | [1](#f1) [2](#f2) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | | `/tag/{tagLabel}` | DELETE | Delete this tag | - | - | 1.1.0 | diff --git a/src/handlers/tag.ts b/src/handlers/tag.ts index 562ac12..dc47706 100644 --- a/src/handlers/tag.ts +++ b/src/handlers/tag.ts @@ -1,6 +1,7 @@ import { respond } from "../helpers/cors.ts"; import { processParams } from "../helpers/processParams.ts"; import * as metafinder from "../meta/finder.ts"; +import * as metastore from "../meta/store.ts"; type TagCreate = { label: string; @@ -11,6 +12,29 @@ type TagCreate = { }; // only for parameterizable tags }; +export async function createTag( + request: Request, + _match: URLPatternResult, +): Promise { + const tagObject: TagCreate = await request.json(); + if (!tagObject?.label) return respond("No label provided", { status: 400 }); + if ( + tagObject?.parameter && + tagObject.parameter.type !== "http://www.w3.org/2001/XMLSchema#decimal" && + tagObject.parameter.type !== "http://www.w3.org/2001/XMLSchema#date" + ) return respond("Invalid type", { status: 400 }); + const tagList = await metafinder.getTagList(); + if (tagList.some((e) => e.label === tagObject.label)) { + return respond("Tag already exists", { status: 400 }); + } + const regex = /\s|^[.]{1,2}$|\/|\\|#|"|'|,|;|:|\?/; + if (regex.test(tagObject.label)) { + return respond("Label contains forbidden characters", { status: 400 }); + } + await metastore.createTag(tagObject.label, tagObject.parameter?.type); + return respond(undefined, { status: 204 }); +} + export async function getDocs( request: Request, match: URLPatternResult, diff --git a/src/meta/store.ts b/src/meta/store.ts index f2ac2ff..ed5826c 100644 --- a/src/meta/store.ts +++ b/src/meta/store.ts @@ -66,6 +66,28 @@ WHERE { OPTIONAL { s:name ?o } }`; return await fusekiUpdate(query); } +export async function createTag( + label: string, + type?: + | "http://www.w3.org/2001/XMLSchema#decimal" + | "http://www.w3.org/2001/XMLSchema#date", +) { + const tagType = type ? "ParameterizableTag" : "Tag"; + const valueType = type ? "tridoc:valueType <" + type + ">;\n" : ""; + const query = ` +PREFIX rdf: +PREFIX xsd: +PREFIX tridoc: +PREFIX s: +INSERT DATA { + GRAPH { + rdf:type tridoc:${tagType} ; + ${valueType} tridoc:label "${escapeLiteral(label)}" . + } +}`; + return await fusekiUpdate(query); +} + export function restore(turtleData: string) { return fusekiUpdate(` CLEAR GRAPH ; diff --git a/src/server/routes.ts b/src/server/routes.ts index 033a089..e00379a 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -70,7 +70,7 @@ export const routes: { handler: doc.postTag, }, { pattern: new URLPattern({ pathname: "/tag" }), - handler: notImplemented, + handler: tag.createTag, }], "PUT": [{ pattern: new URLPattern({ pathname: "/doc/:id/title" }), From 876f0ae5486086587a0561974ac752e9d5829afd Mon Sep 17 00:00:00 2001 From: nleanba Date: Mon, 24 Oct 2022 18:47:55 +0000 Subject: [PATCH 30/35] added DELETE /tag/label and /doc/:id/tag/:label --- README.md | 4 ++-- src/handlers/doc.ts | 11 +++++++++++ src/handlers/tag.ts | 11 +++++++++++ src/meta/delete.ts | 40 ++++++++++++++++++++++++++++++++++++++++ src/server/routes.ts | 4 ++-- 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e1ea603..7c66da2 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | ✅ | | `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | ✅ | | `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | ✅ | -| `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 | +| `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 |✅ | | `/doc/{id}/thumb` | GET | Get document thumbnail | - | PNG (300px wide) | 1.5.0 | ✅ | | `/doc/{id}/title` | PUT | Set document title | `{"title": "the_Title"}` | - | 1.1.0 | ✅ | | `/doc/{id}/title` | GET | Get document title | - | `{"title": "the_Title"}` | 1.1.0 | ✅ | @@ -129,7 +129,7 @@ When getting a comment, a JSON array with objects of the following structure is | `/tag` | POST | Create new tag | See above | - | 1.1.0 | ✅ | | `/tag` | GET | Get (list of) all tags | - | - | 1.1.0 | ✅ | | `/tag/{tagLabel}` | GET | Get Documents with this tag. Same as `/doc?tag={tagLabel}` | [1](#f1) [2](#f2) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | -| `/tag/{tagLabel}` | DELETE | Delete this tag | - | - | 1.1.0 | +| `/tag/{tagLabel}` | DELETE | Delete this tag | - | - | 1.1.0 | ✅ | | `/version` | GET | Get tridoc version | - | semver version number | 1.1.0 | ✅ | #### URL-Parameters supported: diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index e359bc3..401b4ba 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -44,6 +44,17 @@ export async function deleteDoc( return respond(undefined, { status: 204 }); } +export async function deleteTag( + _request: Request, + match: URLPatternResult, +) { + await metadelete.deleteTag( + decodeURIComponent(match.pathname.groups.tagLabel), + match.pathname.groups.id, + ); + return respond(undefined, { status: 204 }); +} + export async function getComments( _request: Request, match: URLPatternResult, diff --git a/src/handlers/tag.ts b/src/handlers/tag.ts index dc47706..8c376bb 100644 --- a/src/handlers/tag.ts +++ b/src/handlers/tag.ts @@ -1,5 +1,6 @@ import { respond } from "../helpers/cors.ts"; import { processParams } from "../helpers/processParams.ts"; +import * as metadelete from "../meta/delete.ts"; import * as metafinder from "../meta/finder.ts"; import * as metastore from "../meta/store.ts"; @@ -35,6 +36,16 @@ export async function createTag( return respond(undefined, { status: 204 }); } +export async function deleteTag( + _request: Request, + match: URLPatternResult, +) { + await metadelete.deleteTag( + decodeURIComponent(match.pathname.groups.tagLabel), + ); + return respond(undefined, { status: 204 }); +} + export async function getDocs( request: Request, match: URLPatternResult, diff --git a/src/meta/delete.ts b/src/meta/delete.ts index 9398486..ba33012 100644 --- a/src/meta/delete.ts +++ b/src/meta/delete.ts @@ -1,5 +1,45 @@ import { fusekiUpdate } from "./fusekiFetch.ts"; +export async function deleteTag(label: string, id?: string) { + await fusekiUpdate(` +PREFIX rdf: +PREFIX s: +PREFIX tridoc: +WITH +DELETE { + ${ + id ? ` tridoc:tag ?ptag + ` : `?ptag ?p ?o . + ?s ?p1 ?ptag` + } +} +WHERE { + ?ptag tridoc:parameterizableTag ?tag. + ?tag tridoc:label "${label}" . + OPTIONAL { ?ptag ?p ?o } + OPTIONAL { + ${id ? ` tridoc:tag ?ptag` : "?s ?p1 ?ptag"} + } +}`); + await fusekiUpdate(` +PREFIX rdf: +PREFIX s: +PREFIX tridoc: +WITH +DELETE { + ${ + id ? ` tridoc:tag ?tag` : `?tag ?p ?o . + ?s ?p1 ?tag` + } +} +WHERE { + ?tag tridoc:label "${label}" . + OPTIONAL { ?tag ?p ?o } + OPTIONAL { + ${id ? ` ?p1 ?tag` : "?s ?p1 ?tag"} + } +}`); +} + export function deleteFile(id: string) { return fusekiUpdate(` WITH diff --git a/src/server/routes.ts b/src/server/routes.ts index e00379a..d784651 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -84,13 +84,13 @@ export const routes: { handler: doc.deleteDoc, }, { pattern: new URLPattern({ pathname: "/doc/:id/tag/:tagLabel" }), - handler: notImplemented, + handler: doc.deleteTag, }, { pattern: new URLPattern({ pathname: "/doc/:id/title" }), handler: notImplemented, }, { pattern: new URLPattern({ pathname: "/tag/:tagLabel" }), - handler: notImplemented, + handler: tag.deleteTag, }, { pattern: new URLPattern({ pathname: "/raw/rdf" }), handler: raw.deleteRDFFile, From aa92beb52e3d2dec8f0762eaca09cbd4973b94a4 Mon Sep 17 00:00:00 2001 From: nleanba Date: Mon, 24 Oct 2022 19:01:23 +0000 Subject: [PATCH 31/35] refined HTTP response codes --- src/handlers/doc.ts | 6 +++--- src/handlers/tag.ts | 2 +- src/meta/delete.ts | 18 ++++++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 401b4ba..007f36f 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -188,7 +188,7 @@ export async function postComment( ): Promise { const id = match.pathname.groups.id; await metastore.addComment(id, (await request.json()).text); - return respond(undefined, { status: 204 }); + return respond(undefined, { status: 201 }); } export async function postPDF( @@ -257,7 +257,7 @@ export async function postTag( return respond("No value provided", { status: 400 }); } await metastore.addTag(id, tagObject.label, tagObject.parameter?.value, type); - return respond(undefined, { status: 204 }); + return respond(undefined, { status: 201 }); } export async function putTitle( @@ -267,5 +267,5 @@ export async function putTitle( const id = match.pathname.groups.id; const title: string = (await request.json())?.title; await metastore.addTitle(id, title); - return respond(undefined, { status: 204 }); + return respond(undefined, { status: 201 }); } diff --git a/src/handlers/tag.ts b/src/handlers/tag.ts index 8c376bb..656547e 100644 --- a/src/handlers/tag.ts +++ b/src/handlers/tag.ts @@ -33,7 +33,7 @@ export async function createTag( return respond("Label contains forbidden characters", { status: 400 }); } await metastore.createTag(tagObject.label, tagObject.parameter?.type); - return respond(undefined, { status: 204 }); + return respond(undefined, { status: 201 }); } export async function deleteTag( diff --git a/src/meta/delete.ts b/src/meta/delete.ts index ba33012..f7a6ddb 100644 --- a/src/meta/delete.ts +++ b/src/meta/delete.ts @@ -1,16 +1,17 @@ import { fusekiUpdate } from "./fusekiFetch.ts"; export async function deleteTag(label: string, id?: string) { - await fusekiUpdate(` + await Promise.allSettled([ + fusekiUpdate(` PREFIX rdf: PREFIX s: PREFIX tridoc: WITH DELETE { ${ - id ? ` tridoc:tag ?ptag + ` : `?ptag ?p ?o . + id ? ` tridoc:tag ?ptag + ` : `?ptag ?p ?o . ?s ?p1 ?ptag` - } + } } WHERE { ?ptag tridoc:parameterizableTag ?tag. @@ -19,17 +20,17 @@ WHERE { OPTIONAL { ${id ? ` tridoc:tag ?ptag` : "?s ?p1 ?ptag"} } -}`); - await fusekiUpdate(` +}`), + fusekiUpdate(` PREFIX rdf: PREFIX s: PREFIX tridoc: WITH DELETE { ${ - id ? ` tridoc:tag ?tag` : `?tag ?p ?o . + id ? ` tridoc:tag ?tag` : `?tag ?p ?o . ?s ?p1 ?tag` - } + } } WHERE { ?tag tridoc:label "${label}" . @@ -37,7 +38,8 @@ WHERE { OPTIONAL { ${id ? ` ?p1 ?tag` : "?s ?p1 ?tag"} } -}`); +}`), + ]); } export function deleteFile(id: string) { From 8c4d889eb5c99f348a1e9445d8b0619f0d0d157a Mon Sep 17 00:00:00 2001 From: nleanba Date: Mon, 24 Oct 2022 19:06:22 +0000 Subject: [PATCH 32/35] implemented DELETE /doc/:id/title --- README.md | 48 ++++++++++++++++++++++---------------------- src/handlers/doc.ts | 8 ++++++++ src/meta/delete.ts | 14 ++++++++++--- src/server/routes.ts | 3 +-- 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 7c66da2..58948b9 100644 --- a/README.md +++ b/README.md @@ -105,32 +105,32 @@ When getting a comment, a JSON array with objects of the following structure is ## API -| Address | Method | Description | Request / Payload | Response | Implemented in Version | deno? | +| Address | Method | Description | Request / Payload | Response | Implemented in Version | | - | - | - | - | - | - | - | -| `/count` | GET | Count (matching) documents | [1](#f1) [3](#f3) | Number | 1.1.0 | ✅ | -| `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | ✅ | -| `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | -| `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | ✅ | -| `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | ✅ | -| `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | ✅ | -| `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | ✅ | -| `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | ✅ | -| `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | ✅ | -| `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 |✅ | -| `/doc/{id}/thumb` | GET | Get document thumbnail | - | PNG (300px wide) | 1.5.0 | ✅ | -| `/doc/{id}/title` | PUT | Set document title | `{"title": "the_Title"}` | - | 1.1.0 | ✅ | -| `/doc/{id}/title` | GET | Get document title | - | `{"title": "the_Title"}` | 1.1.0 | ✅ | +| `/count` | GET | Count (matching) documents | [1](#f1) [3](#f3) | Number | 1.1.0 | +| `/doc` | POST | Add / Store Document | PDF[5](#f5) | - | 1.1.0 | +| `/doc` | GET | Get List of all (matching) documents | [1](#f1) [2](#f2) [3](#f3) | Array of objects with document identifiers and titles (where available) | 1.1.0 | +| `/doc/{id}` | GET | Get this document | - | PDF | 1.1.0 | +| `/doc/{id}` | DELETE | Deletes all metadata associated with the document. Document will not be deleted and is stays accessible over /doc/{id}. | - | - | 1.1.0 | +| `/doc/{id}/comment` | POST | Add comment to document | Comment object / See above | - | 1.2.0 | +| `/doc/{id}/comment` | GET | Get comments | - | Array of comment objects | 1.2.0 | +| `/doc/{id}/tag` | POST | Add a tag to document | Tag object / See above | - | 1.1.0 | +| `/doc/{id}/tag` | GET | Get tags of document | - | Array of tag objects | 1.1.0 | +| `/doc/{id}/tag/{tagLabel}` | DELETE | Remove tag from document | - | - | 1.1.0 | +| `/doc/{id}/thumb` | GET | Get document thumbnail | - | PNG (300px wide) | 1.5.0 | +| `/doc/{id}/title` | PUT | Set document title | `{"title": "the_Title"}` | - | 1.1.0 | +| `/doc/{id}/title` | GET | Get document title | - | `{"title": "the_Title"}` | 1.1.0 | | `/doc/{id}/title` | DELETE | Reset document title | - | - | 1.1.0 | -| `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | ✅ | -| `/raw/rdf` | GET | Get all metadata as RDF. Useful for Backups | [4](#f4) | RDF, Content-Type defined over request Headers or ?accept. Fallback to text/turtle. | 1.1.0 | ✅ | -| `/raw/rdf` | DELETE | "Cancel" failed zip upload—use only if certain it’s done & failed | | | (deno only) | ✅ | -| `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | ✅ | -| `/raw/zip` | PUT | Replace all data with backup zip | ZIP | Replaces the metadata and adds the blobs from the zip | 1.3.0 | ✅ | -| `/tag` | POST | Create new tag | See above | - | 1.1.0 | ✅ | -| `/tag` | GET | Get (list of) all tags | - | - | 1.1.0 | ✅ | -| `/tag/{tagLabel}` | GET | Get Documents with this tag. Same as `/doc?tag={tagLabel}` | [1](#f1) [2](#f2) | Array of objects with document identifiers and titles (where available) | 1.1.0 | ✅ | -| `/tag/{tagLabel}` | DELETE | Delete this tag | - | - | 1.1.0 | ✅ | -| `/version` | GET | Get tridoc version | - | semver version number | 1.1.0 | ✅ | +| `/doc/{id}/meta` | GET | Get various metadata | - | `{"title": "the_Title", "tags":[...], "comments": [...] ... }` | 1.1.0 \| .comments & .created in 1.2.1 | +| `/raw/rdf` | GET | Get all metadata as RDF. Useful for Backups | [4](#f4) | RDF, Content-Type defined over request Headers or ?accept. Fallback to text/turtle. | 1.1.0 | +| `/raw/rdf` | DELETE | "Cancel" failed zip upload—use only if certain it’s done & failed | | | (deno only) | +| `/raw/zip` or `/raw/tgz` | GET | Get all data. Useful for backups | - | ZIP / TGZ containing blobs/ directory with all pdfs as stored within tridoc and a rdf.ttl file with all metadata. | 1.3.0 | +| `/raw/zip` | PUT | Replace all data with backup zip | ZIP | Replaces the metadata and adds the blobs from the zip | 1.3.0 | +| `/tag` | POST | Create new tag | See above | - | 1.1.0 | +| `/tag` | GET | Get (list of) all tags | - | - | 1.1.0 | +| `/tag/{tagLabel}` | GET | Get Documents with this tag. Same as `/doc?tag={tagLabel}` | [1](#f1) [2](#f2) | Array of objects with document identifiers and titles (where available) | 1.1.0 | +| `/tag/{tagLabel}` | DELETE | Delete this tag | - | - | 1.1.0 | +| `/version` | GET | Get tridoc version | - | semver version number | 1.1.0 | #### URL-Parameters supported: diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 007f36f..471d0cb 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -54,6 +54,14 @@ export async function deleteTag( ); return respond(undefined, { status: 204 }); } +export async function deleteTitle( + request: Request, + match: URLPatternResult, +): Promise { + const id = match.pathname.groups.id; + await metadelete.deleteTitle(id); + return respond(undefined, { status: 201 }); +} export async function getComments( _request: Request, diff --git a/src/meta/delete.ts b/src/meta/delete.ts index f7a6ddb..be9e4bc 100644 --- a/src/meta/delete.ts +++ b/src/meta/delete.ts @@ -1,5 +1,12 @@ import { fusekiUpdate } from "./fusekiFetch.ts"; +export function deleteFile(id: string) { + return fusekiUpdate(` +WITH +DELETE { ?p ?o } +WHERE { ?p ?o }`); +} + export async function deleteTag(label: string, id?: string) { await Promise.allSettled([ fusekiUpdate(` @@ -42,9 +49,10 @@ WHERE { ]); } -export function deleteFile(id: string) { +export function deleteTitle(id: string) { return fusekiUpdate(` +PREFIX s: WITH -DELETE { ?p ?o } -WHERE { ?p ?o }`); +DELETE { s:name ?o } +WHERE { s:name ?o }`); } diff --git a/src/server/routes.ts b/src/server/routes.ts index d784651..231fd6c 100644 --- a/src/server/routes.ts +++ b/src/server/routes.ts @@ -1,7 +1,6 @@ import { options } from "../handlers/cors.ts"; import { count } from "../handlers/count.ts"; import * as doc from "../handlers/doc.ts"; -import { notImplemented } from "../handlers/notImplemented.ts"; import * as raw from "../handlers/raw.ts"; import * as tag from "../handlers/tag.ts"; import { version } from "../handlers/version.ts"; @@ -87,7 +86,7 @@ export const routes: { handler: doc.deleteTag, }, { pattern: new URLPattern({ pathname: "/doc/:id/title" }), - handler: notImplemented, + handler: doc.deleteTitle, }, { pattern: new URLPattern({ pathname: "/tag/:tagLabel" }), handler: tag.deleteTag, From 30e234ed326cc89c8c5e38c875432cdb1ee9a569 Mon Sep 17 00:00:00 2001 From: nleanba <25827850+nleanba@users.noreply.github.com> Date: Fri, 13 Jan 2023 19:49:53 +0100 Subject: [PATCH 33/35] added message about file permissions as temp fix --- src/server/server.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/server/server.ts b/src/server/server.ts index 4479ff1..e151cbc 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -40,6 +40,10 @@ const handler = async (request: Request): Promise => { ); return respond("404 Path not found", { status: 404 }); } catch (error) { + let message; + if (error instanceof Deno.errors.PermissionDenied) { + message = "Got “Permission Denied” trying to access the file on disk.\n\n Please run ```docker exec -u 0 [name of backend-container] chmod -R a+r ./blobs/ rdf.ttl``` on the host server to fix this and similar issues for the future." + } console.log( (new Date()).toISOString(), request.method, @@ -47,7 +51,7 @@ const handler = async (request: Request): Promise => { "→ 500: ", error, ); - return respond("500 " + error, { status: 500 }); + return respond("500 " + (message || error), { status: 500 }); } }; From b8752064301e3f807c9d28778bc7fabf89c881b2 Mon Sep 17 00:00:00 2001 From: nleanba <25827850+nleanba@users.noreply.github.com> Date: Sun, 15 Jan 2023 15:28:34 +0100 Subject: [PATCH 34/35] =?UTF-8?q?Added=20correct=20content-type=20for=20al?= =?UTF-8?q?l=20JSON=20responses=20=E2=86=92=20v1.6.0-alpha.deno.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/deps.ts | 2 +- src/handlers/doc.ts | 28 +++++++++++++++++++++++++--- src/handlers/tag.ts | 16 +++++++++++++--- src/server/server.ts | 2 +- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/deps.ts b/src/deps.ts index c2af758..85736e4 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,4 +1,4 @@ -export const VERSION = "1.6.0-alpha.deno"; +export const VERSION = "1.6.0-alpha.deno.1"; export { encode } from "https://deno.land/std@0.160.0/encoding/base64.ts"; export { emptyDir, ensureDir } from "https://deno.land/std@0.160.0/fs/mod.ts"; diff --git a/src/handlers/doc.ts b/src/handlers/doc.ts index 471d0cb..db7b454 100644 --- a/src/handlers/doc.ts +++ b/src/handlers/doc.ts @@ -69,7 +69,11 @@ export async function getComments( ): Promise { const id = match.pathname.groups.id; const response = await metafinder.getComments(id); - return respond(JSON.stringify(response)); + return respond(JSON.stringify(response), { + headers: { + "content-type": "application/json; charset=utf-8", + }, + }); } export async function getPDF( @@ -110,6 +114,11 @@ export async function getMeta( comments: await metafinder.getComments(id), tags: await metafinder.getTags(id), }), + { + headers: { + "content-type": "application/json; charset=utf-8", + }, + }, ); } @@ -118,7 +127,11 @@ export async function getTags( match: URLPatternResult, ): Promise { const id = match.pathname.groups.id; - return respond(JSON.stringify(await metafinder.getTags(id))); + return respond(JSON.stringify(await metafinder.getTags(id)), { + headers: { + "content-type": "application/json; charset=utf-8", + }, + }); } export async function getThumb( @@ -178,6 +191,11 @@ export async function getTitle( const id = match.pathname.groups.id; return respond( JSON.stringify({ title: (await metafinder.getBasicMeta(id)).title }), + { + headers: { + "content-type": "application/json; charset=utf-8", + }, + }, ); } @@ -187,7 +205,11 @@ export async function list( ): Promise { const params = await processParams(request); const response = await metafinder.getDocumentList(params); - return respond(JSON.stringify(response)); + return respond(JSON.stringify(response), { + headers: { + "content-type": "application/json; charset=utf-8", + }, + }); } export async function postComment( diff --git a/src/handlers/tag.ts b/src/handlers/tag.ts index 656547e..27cf821 100644 --- a/src/handlers/tag.ts +++ b/src/handlers/tag.ts @@ -23,7 +23,9 @@ export async function createTag( tagObject?.parameter && tagObject.parameter.type !== "http://www.w3.org/2001/XMLSchema#decimal" && tagObject.parameter.type !== "http://www.w3.org/2001/XMLSchema#date" - ) return respond("Invalid type", { status: 400 }); + ) { + return respond("Invalid type", { status: 400 }); + } const tagList = await metafinder.getTagList(); if (tagList.some((e) => e.label === tagObject.label)) { return respond("Tag already exists", { status: 400 }); @@ -54,12 +56,20 @@ export async function getDocs( tags: [[match.pathname.groups.tagLabel]], }); const response = await metafinder.getDocumentList(params); - return respond(JSON.stringify(response)); + return respond(JSON.stringify(response), { + headers: { + "content-type": "application/json; charset=utf-8", + }, + }); } export async function getTagList( _request: Request, _match: URLPatternResult, ): Promise { - return respond(JSON.stringify(await metafinder.getTagList())); + return respond(JSON.stringify(await metafinder.getTagList()), { + headers: { + "content-type": "application/json; charset=utf-8", + }, + }); } diff --git a/src/server/server.ts b/src/server/server.ts index e151cbc..f810f0a 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -48,7 +48,7 @@ const handler = async (request: Request): Promise => { (new Date()).toISOString(), request.method, path, - "→ 500: ", + "→ 500:", error, ); return respond("500 " + (message || error), { status: 500 }); From 9d17a038e920a6e9186e5bb63f5768e305939748 Mon Sep 17 00:00:00 2001 From: nleanba <25827850+nleanba@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:31:58 +0200 Subject: [PATCH 35/35] Removed deadlock Apparently, waiting for status before waiting for output will wait forever --- src/helpers/pdfprocessor.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/helpers/pdfprocessor.ts b/src/helpers/pdfprocessor.ts index 585b3db..1e267d2 100644 --- a/src/helpers/pdfprocessor.ts +++ b/src/helpers/pdfprocessor.ts @@ -2,7 +2,8 @@ const decoder = new TextDecoder("utf-8"); export async function getText(path: string) { const p = Deno.run({ cmd: ["pdftotext", path, "-"], stdout: "piped" }); + const output = decoder.decode(await p.output()); const { success, code } = await p.status(); - if (!success) throw new Error("pdfsandwich failed with code " + code); - return decoder.decode(await p.output()); + if (!success) throw new Error("pdftotext failed with code " + code); + return output; }