From 4f0dca00e23c98a6c4ed261b7d23298667c37f98 Mon Sep 17 00:00:00 2001 From: Jory Schossau Date: Sat, 28 May 2022 16:38:29 -0400 Subject: [PATCH 1/5] adds linux progress callbacks --- src/puppy.nim | 8 ++++---- src/puppy/platforms/linux/platform.nim | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/puppy.nim b/src/puppy.nim index b20d1f3..8e14b22 100644 --- a/src/puppy.nim +++ b/src/puppy.nim @@ -19,7 +19,7 @@ proc addDefaultHeaders(req: Request) = # If there isn't a specific accept-encoding specified, enable gzip req.headers["accept-encoding"] = "gzip" -proc fetch*(req: Request): Response {.raises: [PuppyError].} = +proc fetch*(req: Request, onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) {.closure} = nil): Response {.raises: [PuppyError].} = if req.url.scheme notin ["http", "https"]: raise newException( PuppyError, "Unsupported request scheme: " & req.url.scheme @@ -30,7 +30,7 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} = if req.timeout == 0: req.timeout = 60 - platform.fetch(req) + platform.fetch(req, onprogress) proc newRequest*( url: string, @@ -45,10 +45,10 @@ proc newRequest*( result.headers = headers result.timeout = timeout -proc fetch*(url: string, headers = newSeq[Header]()): string = +proc fetch*(url: string, headers = newSeq[Header](), onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) = nil): string = let req = newRequest(url, "get", headers) - res = req.fetch() + res = req.fetch(onprogress) if res.code == 200: return res.body raise newException(PuppyError, diff --git a/src/puppy/platforms/linux/platform.nim b/src/puppy/platforms/linux/platform.nim index 9535e1e..dbc3f84 100644 --- a/src/puppy/platforms/linux/platform.nim +++ b/src/puppy/platforms/linux/platform.nim @@ -1,11 +1,19 @@ import libcurl, puppy/common, std/strutils, zippy +from std/sugar import capture type StringWrap = object ## As strings are value objects they need ## some sort of wrapper to be passed to C. str: string -proc fetch*(req: Request): Response {.raises: [PuppyError].} = +proc cProgressCallback(clientp: pointer; dltotal, dlnow, ultotal, ulnow: cdouble): cint {.cdecl, raises: [].} = + try: + cast[proc(dltotal, dlnow, ultotal, ulnow: int) {.nimcall.}](clientp)(dltotal.int, dlnow.int, ultotal.int, ulnow.int) + except: + echo getCurrentExceptionMsg() + quit(1) + +proc fetch*(req: Request, onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) {.closure.} = nil): Response {.raises: [PuppyError].} = result = Response() {.push stackTrace: off.} @@ -39,6 +47,11 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} = discard curl.easy_setopt(OPT_CUSTOMREQUEST, strings[1].cstring) discard curl.easy_setopt(OPT_TIMEOUT, req.timeout.int) + if not isNil onprogress: + discard curl.easy_setopt(OPT_NOPROGRESS, 0) + discard curl.easy_setopt(OPT_PROGRESSDATA, cast[pointer](onprogress.rawProc)) + discard curl.easy_setopt(OPT_PROGRESSFUNCTION, cProgressCallback) + # Create the Pslist for passing headers to curl manually. This is to # avoid needing to call slist_free_all which creates problems var slists: seq[Slist] From a7b5c9cb03be856feae45432b7be9aca5a72770f Mon Sep 17 00:00:00 2001 From: joryschossau Date: Sun, 29 May 2022 18:46:34 -0700 Subject: [PATCH 2/5] adds win dl callback --- src/puppy.nim | 8 ++++---- src/puppy/common.nim | 4 ++++ src/puppy/platforms/win32/platform.nim | 23 ++++++++++++++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/puppy.nim b/src/puppy.nim index 8e14b22..c84b4ab 100644 --- a/src/puppy.nim +++ b/src/puppy.nim @@ -19,7 +19,7 @@ proc addDefaultHeaders(req: Request) = # If there isn't a specific accept-encoding specified, enable gzip req.headers["accept-encoding"] = "gzip" -proc fetch*(req: Request, onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) {.closure} = nil): Response {.raises: [PuppyError].} = +proc fetch*(req: Request, onDLProgress: ProgressCallback = ProgressCallback()): Response {.raises: [PuppyError].} = if req.url.scheme notin ["http", "https"]: raise newException( PuppyError, "Unsupported request scheme: " & req.url.scheme @@ -30,7 +30,7 @@ proc fetch*(req: Request, onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) if req.timeout == 0: req.timeout = 60 - platform.fetch(req, onprogress) + platform.fetch(req, onDLProgress) proc newRequest*( url: string, @@ -45,10 +45,10 @@ proc newRequest*( result.headers = headers result.timeout = timeout -proc fetch*(url: string, headers = newSeq[Header](), onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) = nil): string = +proc fetch*(url: string, headers = newSeq[Header](), onDLProgress: ProgressCallback = ProgressCallback()): string = let req = newRequest(url, "get", headers) - res = req.fetch(onprogress) + res = req.fetch(onDLProgress) if res.code == 200: return res.body raise newException(PuppyError, diff --git a/src/puppy/common.nim b/src/puppy/common.nim index 6a06d9e..2cff07c 100644 --- a/src/puppy/common.nim +++ b/src/puppy/common.nim @@ -23,6 +23,10 @@ type PuppyError* = object of IOError ## Raised if an operation fails. + ProgressCallback* = object + userp*: pointer + callback*: proc(userp: pointer; total, current: int) + proc `[]`*(headers: seq[Header], key: string): string = ## Get a key out of headers. Not case sensitive. ## Use a for loop to get multiple keys. diff --git a/src/puppy/platforms/win32/platform.nim b/src/puppy/platforms/win32/platform.nim index 4c4c422..526e198 100644 --- a/src/puppy/platforms/win32/platform.nim +++ b/src/puppy/platforms/win32/platform.nim @@ -1,6 +1,6 @@ import puppy/common, std/strutils, utils, windefs, zippy -proc fetch*(req: Request): Response {.raises: [PuppyError].} = +proc fetch*(req: Request, onDLProgress: ProgressCallback): Response {.raises: [PuppyError].} = result = Response() var hSession, hConnect, hRequest: HINTERNET @@ -187,6 +187,20 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} = key: parts[0].strip(), value: parts[1].strip() )) + + var contentLength = 0 + block: + var contentLengthValue = "" + try: + for header in result.headers: + if header.key == "Content-Length": + contentLengthValue = header.value + contentLength = parseInt(header.value) + except ValueError: + raise newException( + PuppyError, + "Content-Length error: '" & contentLengthValue & "' not an integer." + ) var i: int result.body.setLen(8192) @@ -208,6 +222,13 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} = if bytesRead == 0: break + if not isNil onDLProgress.callback: + try: + onDLProgress.callback(onDLProgress.userp, contentLength, i) + except: + raise newException( + PuppyError, "ProgressCallback invalid userp pointer" + ) if i == result.body.len: result.body.setLen(min(i * 2, i + 100 * 1024 * 1024)) From b3990699ab0f202d141b2a28f891ce4bd7ebd4f5 Mon Sep 17 00:00:00 2001 From: Jory Schossau Date: Sun, 29 May 2022 22:16:42 -0400 Subject: [PATCH 3/5] aligns linux and windows callback APIs --- src/puppy/platforms/linux/platform.nim | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/puppy/platforms/linux/platform.nim b/src/puppy/platforms/linux/platform.nim index dbc3f84..09d5a8b 100644 --- a/src/puppy/platforms/linux/platform.nim +++ b/src/puppy/platforms/linux/platform.nim @@ -1,5 +1,7 @@ -import libcurl, puppy/common, std/strutils, zippy +import libcurl except Progress_callback +import puppy/common, std/strutils, zippy from std/sugar import capture +from std/decls import byaddr type StringWrap = object ## As strings are value objects they need @@ -8,12 +10,13 @@ type StringWrap = object proc cProgressCallback(clientp: pointer; dltotal, dlnow, ultotal, ulnow: cdouble): cint {.cdecl, raises: [].} = try: - cast[proc(dltotal, dlnow, ultotal, ulnow: int) {.nimcall.}](clientp)(dltotal.int, dlnow.int, ultotal.int, ulnow.int) + let cbPtr = cast[ptr ProgressCallback](clientp) + cbPtr.callback(cbPtr.userp, dltotal.int, dlnow.int) except: echo getCurrentExceptionMsg() quit(1) -proc fetch*(req: Request, onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) {.closure.} = nil): Response {.raises: [PuppyError].} = +proc fetch*(req: Request, onDLProgress: ProgressCallback): Response {.raises: [PuppyError].} = result = Response() {.push stackTrace: off.} @@ -47,9 +50,12 @@ proc fetch*(req: Request, onprogress: proc(dltotal, dlnow, ultotal, ulnow: int) discard curl.easy_setopt(OPT_CUSTOMREQUEST, strings[1].cstring) discard curl.easy_setopt(OPT_TIMEOUT, req.timeout.int) - if not isNil onprogress: + if not isNil onDLProgress.callback: discard curl.easy_setopt(OPT_NOPROGRESS, 0) - discard curl.easy_setopt(OPT_PROGRESSDATA, cast[pointer](onprogress.rawProc)) + #discard curl.easy_setopt(OPT_PROGRESSDATA, cast[pointer](onprogress.rawProc)) + #discard curl.easy_setopt(OPT_PROGRESSFUNCTION, cProgressCallback) + discard curl.easy_setopt(OPT_PROGRESSDATA, cast[pointer](unsafeAddr onDLProgress)) + #discard curl.easy_setopt(OPT_PROGRESSFUNCTION, proc(clientp: pointer, dlt,dln,ult,uln: cdouble): cint = onprogress(clientp, dlt.int,dln.int)) discard curl.easy_setopt(OPT_PROGRESSFUNCTION, cProgressCallback) # Create the Pslist for passing headers to curl manually. This is to From dd276b8c15c160ae714b8e36480333f469e44477 Mon Sep 17 00:00:00 2001 From: Jory Schossau Date: Sun, 29 May 2022 22:20:17 -0400 Subject: [PATCH 4/5] renames userp to clientp to adhere to convention --- src/puppy/common.nim | 4 ++-- src/puppy/platforms/linux/platform.nim | 2 +- src/puppy/platforms/win32/platform.nim | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/puppy/common.nim b/src/puppy/common.nim index 2cff07c..dd5d22a 100644 --- a/src/puppy/common.nim +++ b/src/puppy/common.nim @@ -24,8 +24,8 @@ type PuppyError* = object of IOError ## Raised if an operation fails. ProgressCallback* = object - userp*: pointer - callback*: proc(userp: pointer; total, current: int) + clientp*: pointer + callback*: proc(clientp: pointer; total, current: int) proc `[]`*(headers: seq[Header], key: string): string = ## Get a key out of headers. Not case sensitive. diff --git a/src/puppy/platforms/linux/platform.nim b/src/puppy/platforms/linux/platform.nim index 09d5a8b..ad410b0 100644 --- a/src/puppy/platforms/linux/platform.nim +++ b/src/puppy/platforms/linux/platform.nim @@ -11,7 +11,7 @@ type StringWrap = object proc cProgressCallback(clientp: pointer; dltotal, dlnow, ultotal, ulnow: cdouble): cint {.cdecl, raises: [].} = try: let cbPtr = cast[ptr ProgressCallback](clientp) - cbPtr.callback(cbPtr.userp, dltotal.int, dlnow.int) + cbPtr.callback(cbPtr.clientp, dltotal.int, dlnow.int) except: echo getCurrentExceptionMsg() quit(1) diff --git a/src/puppy/platforms/win32/platform.nim b/src/puppy/platforms/win32/platform.nim index 526e198..bf4d108 100644 --- a/src/puppy/platforms/win32/platform.nim +++ b/src/puppy/platforms/win32/platform.nim @@ -224,10 +224,10 @@ proc fetch*(req: Request, onDLProgress: ProgressCallback): Response {.raises: [P if not isNil onDLProgress.callback: try: - onDLProgress.callback(onDLProgress.userp, contentLength, i) + onDLProgress.callback(onDLProgress.clientp, contentLength, i) except: raise newException( - PuppyError, "ProgressCallback invalid userp pointer" + PuppyError, "ProgressCallback invalid clientp pointer" ) if i == result.body.len: result.body.setLen(min(i * 2, i + 100 * 1024 * 1024)) From 8fc0986ad6b39e3bff4aa7351ae44e7cca06219b Mon Sep 17 00:00:00 2001 From: Jory Schossau Date: Sun, 29 May 2022 22:25:03 -0400 Subject: [PATCH 5/5] adds readme example for dl callbacks --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index f482f97..9bdf355 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,17 @@ echo res.headers echo res.body.len ``` +Get download progress? + +```nim +import puppy + +proc reportProgress(userp: pointer; total, current: int) = + echo total,' ',current + +echo fetch("http://neverssl.com/", onDLProgress = ProgressCallback(callback: reportProgress)) +``` + # Always use Libcurl You can pass `-d:puppyLibcurl` to force use of `libcurl` even on windows and macOS. This is useful to debug, if the some reason native OS API does not work. Libcurl is usually installed on macOS but requires a `curl.dll` on windows.