From ddb3c675d904dbb8f7ca24c65170f51d73b391cd Mon Sep 17 00:00:00 2001 From: J0sueTM Date: Tue, 17 Sep 2024 17:07:19 -0300 Subject: [PATCH 1/3] chore: bump dep versions --- deps.edn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps.edn b/deps.edn index 4c31d91..703a3c0 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,7 @@ {:paths ["resources" "src"] :deps {org.clojure/clojure {:mvn/version "1.11.1"} - com.moclojer/moclojer {:mvn/version "0.3.2"} - com.moclojer/rq {:mvn/version "0.1.4"} + com.moclojer/moclojer {:mvn/version "0.3.4"} + com.moclojer/rq {:mvn/version "0.2.0"} clj-http/clj-http {:mvn/version "3.12.3"} com.zaxxer/HikariCP {:mvn/version "5.0.1"} com.stuartsierra/component {:mvn/version "1.1.0"} From 2c75b6b93ead9597c0e963d7ba1f3b5d3e8f4e43 Mon Sep 17 00:00:00 2001 From: J0sueTM Date: Fri, 20 Sep 2024 17:11:42 -0300 Subject: [PATCH 2/3] chore: bump clj-rq to `v0.2.1` --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 703a3c0..02c5998 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,7 @@ {:paths ["resources" "src"] :deps {org.clojure/clojure {:mvn/version "1.11.1"} com.moclojer/moclojer {:mvn/version "0.3.4"} - com.moclojer/rq {:mvn/version "0.2.0"} + com.moclojer/rq {:mvn/version "0.2.1"} clj-http/clj-http {:mvn/version "3.12.3"} com.zaxxer/HikariCP {:mvn/version "5.0.1"} com.stuartsierra/component {:mvn/version "1.1.0"} From de96413547e3b9368142c0fbeba5b36394f65766 Mon Sep 17 00:00:00 2001 From: j0suetm Date: Mon, 30 Sep 2024 17:24:47 -0300 Subject: [PATCH 3/3] feat: switch to reitit --- .gitignore | 1 + deps.edn | 80 ++++++------ docker/docker-compose.yml | 12 ++ src/com/moclojer/components/core.clj | 2 +- src/com/moclojer/components/moclojer.clj | 5 +- src/com/moclojer/components/mq.clj | 1 + src/com/moclojer/components/router.clj | 143 ++++++++++++---------- src/com/moclojer/components/webserver.clj | 94 ++------------ 8 files changed, 141 insertions(+), 197 deletions(-) diff --git a/.gitignore b/.gitignore index f3f40bb..7342968 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ output.calva-repl /.clj-kondo .envrc redis +localstack diff --git a/deps.edn b/deps.edn index 02c5998..77f3d8d 100644 --- a/deps.edn +++ b/deps.edn @@ -1,33 +1,31 @@ {:paths ["resources" "src"] - :deps {org.clojure/clojure {:mvn/version "1.11.1"} - com.moclojer/moclojer {:mvn/version "0.3.4"} - com.moclojer/rq {:mvn/version "0.2.1"} - clj-http/clj-http {:mvn/version "3.12.3"} - com.zaxxer/HikariCP {:mvn/version "5.0.1"} - com.stuartsierra/component {:mvn/version "1.1.0"} - com.taoensso/timbre {:mvn/version "6.3.1"} - viesti/timbre-json-appender {:mvn/version "0.2.13"} - com.fzakaria/slf4j-timbre {:mvn/version "0.3.21"} - aero/aero {:mvn/version "1.1.6"} - metosin/muuntaja {:mvn/version "0.6.8"} - metosin/malli {:mvn/version "0.11.0"} - metosin/reitit {:mvn/version "0.5.18"} - metosin/reitit-pedestal {:mvn/version "0.5.18"} - metosin/reitit-swagger {:mvn/version "0.5.18"} - metosin/reitit-swagger-ui {:mvn/version "0.5.18"} - io.sentry/sentry {:mvn/version "7.6.0"} - io.sentry/sentry-clj {:mvn/version "7.6.215"} - com.cognitect.aws/api {:mvn/version "0.8.561"} - com.cognitect.aws/endpoints {:mvn/version "1.1.12.230"} - com.cognitect.aws/s3 {:mvn/version "822.2.1145.0"} - org.postgresql/postgresql {:mvn/version "42.6.0"} - com.github.seancorfield/next.jdbc {:mvn/version "1.3.834"} - io.pedestal/pedestal.jetty {:mvn/version "0.5.10"} - io.pedestal/pedestal.service {:mvn/version "0.5.10"} - commons-io/commons-io {:mvn/version "2.16.1"} - commons-fileupload/commons-fileupload {:mvn/version "1.5"} - migratus/migratus {:mvn/version "1.5.1"} - redis.clients/jedis {:mvn/version "5.1.2"}} + :deps {org.clojure/clojure {:mvn/version "1.11.1"} + com.moclojer/moclojer {:mvn/version "0.3.4"} + com.moclojer/rq {:mvn/version "0.2.1"} + clj-http/clj-http {:mvn/version "3.12.3"} + com.zaxxer/HikariCP {:mvn/version "5.0.1"} + com.stuartsierra/component {:mvn/version "1.1.0"} + com.taoensso/timbre {:mvn/version "6.3.1"} + viesti/timbre-json-appender {:mvn/version "0.2.13"} + com.fzakaria/slf4j-timbre {:mvn/version "0.3.21"} + aero/aero {:mvn/version "1.1.6"} + metosin/muuntaja {:mvn/version "0.6.8"} + metosin/malli {:mvn/version "0.11.0"} + metosin/reitit {:mvn/version "0.5.18"} + metosin/reitit-swagger {:mvn/version "0.5.18"} + metosin/reitit-swagger-ui {:mvn/version "0.5.18"} + ring/ring-jetty-adapter {:mvn/version "1.12.2"} + io.sentry/sentry {:mvn/version "7.6.0"} + io.sentry/sentry-clj {:mvn/version "7.6.215"} + com.cognitect.aws/api {:mvn/version "0.8.561"} + com.cognitect.aws/endpoints {:mvn/version "1.1.12.230"} + com.cognitect.aws/s3 {:mvn/version "822.2.1145.0"} + org.postgresql/postgresql {:mvn/version "42.6.0"} + com.github.seancorfield/next.jdbc {:mvn/version "1.3.834"} + commons-io/commons-io {:mvn/version "2.16.1"} + commons-fileupload/commons-fileupload {:mvn/version "1.5"} + migratus/migratus {:mvn/version "1.5.1"} + redis.clients/jedis {:mvn/version "5.1.2"}} :aliases {;; clj -A:dev -m com.moclojer.components :dev @@ -45,17 +43,17 @@ ;; clj -M:test -n com.moclojer.rq.excel-test :test {:extra-paths ["test"] - :extra-deps {org.clojars.bigsy/pg-embedded-clj {:mvn/version "1.0.0"} - lambdaisland/kaocha {:mvn/version "1.70.1086"} - lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"} - nubank/matcher-combinators {:mvn/version "3.5.1"} - nubank/state-flow {:mvn/version "5.14.2"}} - :main-opts ["-m" "kaocha.runner" "--no-capture-output"]} + :extra-deps {org.clojars.bigsy/pg-embedded-clj {:mvn/version "1.0.0"} + lambdaisland/kaocha {:mvn/version "1.70.1086"} + lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"} + nubank/matcher-combinators {:mvn/version "3.5.1"} + nubank/state-flow {:mvn/version "5.14.2"}} + :main-opts ["-m" "kaocha.runner" "--no-capture-output"]} ;; clj -M:nrepl :nrepl {:extra-deps {cider/cider-nrepl {:mvn/version "0.30.0"}} - :main-opts ["-m" "nrepl.cmdline" "--middleware" "[cider.nrepl/cider-middleware]"]} + :main-opts ["-m" "nrepl.cmdline" "--middleware" "[cider.nrepl/cider-middleware]"]} ;; Lint the source ;; clj -M:lint @@ -68,8 +66,8 @@ ;; clj -X:deploy-clojars :deploy-clojars {:extra-deps {slipset/deps-deploy {:mvn/version "RELEASE"}} - :exec-fn deps-deploy.deps-deploy/deploy - :exec-args {:installer :remote - :sign-releases? false - :pom-file "target/classes/META-INF/maven/com.moclojer/components/pom.xml" - :artifact "target/com.moclojer.components.jar"}}}} + :exec-fn deps-deploy.deps-deploy/deploy + :exec-args {:installer :remote + :sign-releases? false + :pom-file "target/classes/META-INF/maven/com.moclojer/components/pom.xml" + :artifact "target/com.moclojer.components.jar"}}}} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index dffb9e0..23aa9cb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,3 +5,15 @@ services: - "6379:6379" volumes: - ./redis:/redis + # https://docs.localstack.cloud/user-guide/aws/s3/#s3-docker-image + localstack: + image: localstack/localstack:s3-latest + ports: + - "127.0.0.1:4566:4566" # Gateway + environment: + - AWS_ACCESS_KEY_ID="foo" + - AWS_SECRET_ACCESS_KEY="bar" + - AWS_DEFAULT_REGION=us-east-1 + - DEBUG=${DEBUG:-1} + volumes: + - "${LOCALSTACK_DIR:-./localstack}:/var/lib/localstack" diff --git a/src/com/moclojer/components/core.clj b/src/com/moclojer/components/core.clj index 5dd2db1..b94d214 100644 --- a/src/com/moclojer/components/core.clj +++ b/src/com/moclojer/components/core.clj @@ -43,7 +43,7 @@ (defn new-router [routes] - (router/map->Router {:router (router/router routes)})) + (router/map->Router {:routes routes})) (defn new-sentry [] (sentry/map->Sentry {})) diff --git a/src/com/moclojer/components/moclojer.clj b/src/com/moclojer/components/moclojer.clj index fe83176..9a7c083 100644 --- a/src/com/moclojer/components/moclojer.clj +++ b/src/com/moclojer/components/moclojer.clj @@ -4,8 +4,7 @@ [com.moclojer.components.logs :as logs] [com.moclojer.io-utils :as m.io-utils] [com.moclojer.server :as server] - [com.stuartsierra.component :as component] - [io.pedestal.http :as http])) + [com.stuartsierra.component :as component])) (defn moclojer-server! [{:keys [config-path join?]}] (let [*router (m.adapters/generate-routes (m.io-utils/open-file config-path))] @@ -28,5 +27,5 @@ (stop [this] (let [stop-fn (-> this :moclojer :stop-future)] (stop-fn) - (http/stop (-> this :moclojer :server)) + (.stop (-> this :moclojer :server)) (assoc this :moclojer nil)))) diff --git a/src/com/moclojer/components/mq.clj b/src/com/moclojer/components/mq.clj index 9f938a9..a791551 100644 --- a/src/com/moclojer/components/mq.clj +++ b/src/com/moclojer/components/mq.clj @@ -2,6 +2,7 @@ (:require [com.moclojer.components.logs :as logs] [com.moclojer.components.sentry :as sentry] + [com.moclojer.rq.queue] [com.moclojer.rq :as rq] [com.moclojer.rq.pubsub :as rq-pubsub] [com.stuartsierra.component :as component])) diff --git a/src/com/moclojer/components/router.clj b/src/com/moclojer/components/router.clj index b1fb82b..97ae329 100644 --- a/src/com/moclojer/components/router.clj +++ b/src/com/moclojer/components/router.clj @@ -1,19 +1,17 @@ (ns com.moclojer.components.router (:require + [clojure.string :as str] [com.moclojer.components.logs :as logs] [com.moclojer.components.sentry :as sentry] [com.stuartsierra.component :as component] [muuntaja.core :as m] [reitit.coercion.malli :as reitit.malli] [reitit.dev.pretty :as pretty] - [reitit.http :as http] - [reitit.http.coercion :as coercion] - [reitit.http.interceptors.exception :as exception] - [reitit.http.interceptors.multipart :as multipart] - [reitit.http.interceptors.muuntaja :as muuntaja] - [reitit.http.interceptors.parameters :as parameters] - [reitit.pedestal :as pedestal] [reitit.ring :as ring] + [reitit.ring.coercion :as coercion] + [reitit.ring.middleware.multipart :as multipart] + [reitit.ring.middleware.muuntaja :as muuntaja] + [reitit.ring.middleware.parameters :as parameters] [reitit.swagger :as swagger] [reitit.swagger-ui :as swagger-ui])) @@ -22,71 +20,80 @@ (sentry/send-event! sentry-cmp {:throwable ex}) (logs/log :error "failed to send sentry event (nil component)"))) -(defn- coercion-error-handler [status] - (fn [exception request] - (logs/log :error "failed to coerce req/resp" - :ctx {:ex-message (.getMessage exception) - :coercion (:errors (ex-data exception))}) - (send-sentry-evt-from-req! request exception) - {:status status - :body (if (= 400 status) - (str "Invalid path or request parameters, with the following errors: " - (:errors (ex-data exception))) - "Error checking path or request parameters.")})) +(defn exception-middleware + [handler-fn] + (fn [request] + (try + (handler-fn request) + (catch Exception e + (logs/log :error (.getMessage e) + :ctx {:exception e}) + (send-sentry-evt-from-req! request e) + {:status 500 + :body (.getMessage e)})))) -(defn- exception-info-handler [exception request] - (logs/log :error "server exception" - :ctx {:ex-message (.getMessage exception)}) - (send-sentry-evt-from-req! request exception) - {:status 500 - :body "Internal error."}) +(defn log-request-middleware + [handler-fn] + (fn [request] + (let [method (str/upper-case (name (:request-method request)))] + (logs/log :info (str method " " (:uri request)) + :ctx {:method method + :host (:server-name request) + :uri (:uri request) + :query-string (:query-string request)}) + (handler-fn request)))) -(def router-settings - {;:reitit.interceptor/transform dev/print-context-diffs ;; pretty context diffs - ;;:validate spec/validate ;; enable spec validation for route data - ;;:reitit.spec/wrap spell/closed ;; strict top-level validation - :exception pretty/exception +(defn cid-middleware + "Extends incoming request with CID if not given already" + [handler-fn] + (fn [request] + (->> {:cid (get-in request [:headers "cid"] + (:cid (logs/gen-ctx-with-cid)))} + (assoc request :ctx) + (handler-fn)))) + +(defn build-components-middleware + [components] + (fn [handler-fn] + (fn [request] + (handler-fn (assoc request :components components))))) + +(defn build-router-settings + [components] + {:exception pretty/exception :data {:coercion reitit.malli/coercion :muuntaja (m/create (-> m/default-options - (assoc-in [:formats "application/json" :decoder-opts :bigdecimals] true))) - :interceptors [;; swagger feature - swagger/swagger-feature - ;; query-params & form-params - (parameters/parameters-interceptor) - ;; content-negotiation - (muuntaja/format-negotiate-interceptor) - ;; encoding response body - (muuntaja/format-response-interceptor) - ;; exception handling - (exception/exception-interceptor - (merge - exception/default-handlers - {:reitit.coercion/request-coercion (coercion-error-handler 400) - :reitit.coercion/response-coercion (coercion-error-handler 500) - clojure.lang.ExceptionInfo exception-info-handler})) - ;; decoding request body - (muuntaja/format-request-interceptor) - ;; coercing response bodys - (coercion/coerce-response-interceptor) - ;; coercing request parameters - (coercion/coerce-request-interceptor) - ;; multipart - (multipart/multipart-interceptor)]}}) - -(defn router [routes] - (pedestal/routing-interceptor - (http/router routes router-settings) - ;; optional default ring handler (if no routes have matched) - (ring/routes - (swagger-ui/create-swagger-ui-handler - {:path "/" - :config {:validatorUrl nil - :operationsSorter "alpha"}}) - (ring/create-resource-handler) - (ring/create-default-handler)))) + (assoc-in [:formats "application/json" + :decoder-opts :bigdecimals] + true))) + :middleware [(build-components-middleware components) + log-request-middleware + cid-middleware + exception-middleware + swagger/swagger-feature + parameters/parameters-middleware + muuntaja/format-negotiate-middleware + muuntaja/format-response-middleware + muuntaja/format-request-middleware + coercion/coerce-response-middleware + coercion/coerce-request-middleware + multipart/multipart-middleware]}}) -(defrecord Router [router] +(defrecord Router [routes] component/Lifecycle - (start [this] this) - (stop [this] this)) + (start [this] + (assoc this :router + (ring/ring-handler + (ring/router + routes + (build-router-settings this)) + (ring/routes + (swagger-ui/create-swagger-ui-handler + {:path "/" + :config {:validatorUrl nil + :operationsSorter "alpha"}}) + (ring/create-resource-handler) + (ring/create-default-handler))))) + (stop [this] + (dissoc this :router))) diff --git a/src/com/moclojer/components/webserver.clj b/src/com/moclojer/components/webserver.clj index 2a694c8..af2841b 100644 --- a/src/com/moclojer/components/webserver.clj +++ b/src/com/moclojer/components/webserver.clj @@ -1,98 +1,24 @@ (ns com.moclojer.components.webserver (:require - [clojure.pprint :refer [pprint]] - [clojure.string :as str] + [ring.adapter.jetty :as jetty] [com.moclojer.components.logs :as logs] - [com.stuartsierra.component :as component] - [io.pedestal.http :as server] - [io.pedestal.interceptor.helpers :refer [before on-request]] - [reitit.pedestal :as pedestal])) - -(defn- add-system [service] - (before (fn [context] (assoc-in context [:request :components] service)))) - -(defn system-interceptors - "Extend to service's interceptors to include one to inject the components - into the request object" - [service-map service] - (update-in service-map - [::server/interceptors] - #(vec (->> % (cons (add-system service)))))) - -(defn cid-interceptor - "Extends incoming request with CID if not given already" - [service-map] - (update-in service-map - [::server/interceptors] - #(conj % (before - (fn [ctx] - (assoc-in ctx [:request :ctx] - {:cid (get-in ctx [:request :headers "cid"] - (:cid (logs/gen-ctx-with-cid)))})))))) - -(def req-logger - (on-request - ::log-request - (fn [req] - (logs/log - :info (str (str/lower-case (name (:request-method req))) " " (:uri req)) - :ctx (apply dissoc req [:body :components :servlet :servlet-request - :servlet-response])) - req))) - -(defn base-service [port] - {::server/port port - ::server/type :jetty - ::server/host "0.0.0.0" - ;; dev false to not lock repl/thread - ::server/join? true - ;; no pedestal routes - ::server/request-logger req-logger - ::server/routes [] - ;; allow serving the swagger-ui styles & scripts from self - ::server/secure-headers {:content-security-policy-settings - {:default-src "'self'" - :style-src "'self' 'unsafe-inline'" - :script-src "'self' 'unsafe-inline'" - :img-src "'self' 'unsafe-inline' data: https://validator.swagger.io"}}}) - -(defn dev-init [service-map router] - (-> service-map - (merge {:env :dev - ;; do not block thread that starts web server - ::server/join? false - ::server/allowed-origins {:creds true :allowed-origins (constantly true)} - ;; Content Security Policy (CSP) is mostly turned off in dev mode - ::server/secure-headers {:content-security-policy-settings {:object-src "none"}}}) - ;; Wire up interceptor chains - (server/default-interceptors) - (pedestal/replace-last-interceptor router) - (server/dev-interceptors))) - -(defn prod-init [service-map router] - (-> service-map - (merge {:env :prod}) - (server/default-interceptors) - (pedestal/replace-last-interceptor router))) + [com.stuartsierra.component :as component])) (defrecord WebServer [config router] component/Lifecycle (start [this] (let [{:webserver/keys [port] :keys [env]} (:config config) - init-fn (if (= env :dev) dev-init prod-init)] + cfg {:port port + :host "0.0.0.0" + :join? true + :env env}] (logs/log :info "starting webserver config" :ctx {:env env :port port}) - (assoc this :webserver - (-> (base-service port) - (init-fn (:router router)) - (cid-interceptor) - (system-interceptors this) - (server/create-server) - (server/start))))) + (assoc this :webserver (jetty/run-jetty (:router router) cfg)))) (stop [this] (logs/log :info "stopping webserver") - (server/stop (:webserver this)) - (dissoc this :webserver) - this)) + (when-let [srv (:webserver this)] + (.stop srv)) + (dissoc this :webserver)))