From 690b18856c6ff7ab85823867b9ee533da0bde1eb Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Mon, 12 Sep 2016 18:32:41 +0300 Subject: [PATCH 01/14] add CLNT feature support * some servers require it to enable features (such as UTF8 support) --- client.go | 6 ++++++ persistent_connection.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/client.go b/client.go index e488d07..7862261 100644 --- a/client.go +++ b/client.go @@ -417,6 +417,12 @@ func (c *Client) openConn(idx int, host string) (pconn *persistentConn, err erro goto Error } + if pconn.hasFeature("CLNT") { + if err = pconn.setClient(); err != nil { + goto Error + } + } + c.mu.Lock() defer c.mu.Unlock() diff --git a/persistent_connection.go b/persistent_connection.go index 2b8d7b3..4aad853 100644 --- a/persistent_connection.go +++ b/persistent_connection.go @@ -228,6 +228,20 @@ func (pconn *persistentConn) hasFeatureWithArg(name, arg string) bool { return found && strings.ToUpper(arg) == val } +func (pconn *persistentConn) setClient() error { + code, msg, err := pconn.sendCommand("CLNT GoFTP") + if err != nil { + return err + } + + if !positiveCompletionReply(code) { + pconn.debug("server doesn't support CLNT: %d-%s", code, msg) + return nil + } + + return nil +} + func (pconn *persistentConn) logIn() error { if pconn.config.User == "" { return nil From eddfb120c58244396163362b761aac9ee7ee2b7e Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Mon, 12 Sep 2016 18:40:21 +0300 Subject: [PATCH 02/14] add UTF8 feature support * while some servers use UTF8 by default if supported, others require activation (and CLNT) * without UTF8 enabled you cannot access files/paths with non-english names as the server will treat your input as ASCII --- client.go | 6 ++++++ persistent_connection.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/client.go b/client.go index 7862261..33b2e54 100644 --- a/client.go +++ b/client.go @@ -423,6 +423,12 @@ func (c *Client) openConn(idx int, host string) (pconn *persistentConn, err erro } } + if pconn.hasFeature("UTF8") { + if err = pconn.setUnicode(); err != nil { + goto Error + } + } + c.mu.Lock() defer c.mu.Unlock() diff --git a/persistent_connection.go b/persistent_connection.go index 4aad853..ba5bf73 100644 --- a/persistent_connection.go +++ b/persistent_connection.go @@ -242,6 +242,20 @@ func (pconn *persistentConn) setClient() error { return nil } +func (pconn *persistentConn) setUnicode() error { + code, msg, err := pconn.sendCommand("OPTS UTF8 ON") + if err != nil { + return err + } + + if !positiveCompletionReply(code) { + pconn.debug("server doesn't support UTF8: %d-%s", code, msg) + return nil + } + + return nil +} + func (pconn *persistentConn) logIn() error { if pconn.config.User == "" { return nil From a1217105bbd5fcadd612dc4f138aa98d7abb90d7 Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Mon, 12 Sep 2016 18:56:25 +0300 Subject: [PATCH 03/14] add MLST feature support (rfc3659) * request only information we can process in parseMLST --- client.go | 6 ++++++ persistent_connection.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/client.go b/client.go index 33b2e54..f4cd922 100644 --- a/client.go +++ b/client.go @@ -429,6 +429,12 @@ func (c *Client) openConn(idx int, host string) (pconn *persistentConn, err erro } } + if pconn.hasFeature("MLST") { + if err = pconn.setMLST(); err != nil { + goto Error + } + } + c.mu.Lock() defer c.mu.Unlock() diff --git a/persistent_connection.go b/persistent_connection.go index ba5bf73..39afe5e 100644 --- a/persistent_connection.go +++ b/persistent_connection.go @@ -256,6 +256,20 @@ func (pconn *persistentConn) setUnicode() error { return nil } +func (pconn *persistentConn) setMLST() error { + code, msg, err := pconn.sendCommand("OPTS MLST type;size;modify;perm;unix.mode;") + if err != nil { + return err + } + + if !positiveCompletionReply(code) { + pconn.debug("server doesn't support UTF8: %d-%s", code, msg) + return nil + } + + return nil +} + func (pconn *persistentConn) logIn() error { if pconn.config.User == "" { return nil From 86961aef897e1ef547741c6c60928797b39803b0 Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Mon, 12 Sep 2016 19:10:55 +0300 Subject: [PATCH 04/14] fix panic when server send empty line in FEAT response --- persistent_connection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistent_connection.go b/persistent_connection.go index 39afe5e..0020e66 100644 --- a/persistent_connection.go +++ b/persistent_connection.go @@ -205,7 +205,7 @@ func (pconn *persistentConn) fetchFeatures() error { } for _, line := range strings.Split(msg, "\n") { - if line[0] == ' ' { + if len(line) > 0 && line[0] == ' ' { parts := strings.SplitN(strings.TrimSpace(line), " ", 2) if len(parts) == 1 { pconn.features[strings.ToUpper(parts[0])] = "" From 89378c2c756d6be5c326d8ea7507e668b108e264 Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Mon, 12 Sep 2016 19:26:52 +0300 Subject: [PATCH 05/14] fix parseMLST issues * support for filenames containing "; " * fix for broken servers sending type "dir" with name "." or ".." --- file_system.go | 4 ++-- persistent_connection.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/file_system.go b/file_system.go index d80bf40..26af53f 100644 --- a/file_system.go +++ b/file_system.go @@ -400,7 +400,7 @@ func parseMLST(entry string, skipSelfParent bool) (os.FileInfo, error) { parseError := ftpError{err: fmt.Errorf(`failed parsing MLST entry: %s`, entry)} incompleteError := ftpError{err: fmt.Errorf(`MLST entry incomplete: %s`, entry)} - parts := strings.Split(entry, "; ") + parts := strings.SplitN(entry, "; ", 2) if len(parts) != 2 { return nil, parseError } @@ -420,7 +420,7 @@ func parseMLST(entry string, skipSelfParent bool) (os.FileInfo, error) { return nil, incompleteError } - if skipSelfParent && (typ == "cdir" || typ == "pdir" || typ == "." || typ == "..") { + if skipSelfParent && (typ == "cdir" || typ == "pdir" || typ == "." || typ == ".."|| parts[1] == "." || parts[1] == "..") { return nil, nil } diff --git a/persistent_connection.go b/persistent_connection.go index 0020e66..7b0b4bc 100644 --- a/persistent_connection.go +++ b/persistent_connection.go @@ -63,6 +63,9 @@ type persistentConn struct { // remember EPSV support epsvNotSupported bool + // remember MLSD support + mlsdNotSupported bool + // tracks the current type (e.g. ASCII/Image) of connection currentType string From adaab2c83669ee76583c6ac760c3e5f3a8cae9b4 Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Mon, 12 Sep 2016 22:20:41 +0300 Subject: [PATCH 06/14] add "dir"-style LIST response support --- file_system.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/file_system.go b/file_system.go index 26af53f..e22a7f2 100644 --- a/file_system.go +++ b/file_system.go @@ -321,6 +321,66 @@ func (f *ftpFile) Sys() interface{} { return f.raw } +var dirRegex = regexp.MustCompile(`^\s*(\S+\s+\S+\s{0,1}\S{2})\s+(\S+)\s+(.*)`) +var dirTimeFormats = []string{ + "01-02-06 03:04", + "01-02-06 03:04PM", + "2006-01-02 15:04", +} + +// 08/03/2016 17:13 directory-name +// 08/07/2016 17:20 1,135 file-name +// or +// 02-10-16 12:10PM 1067784 file.exe +func parseDirLIST(entry string, loc *time.Location, skipSelfParent bool) (os.FileInfo, error) { + matches := dirRegex.FindStringSubmatch(entry) + + if len(matches) == 0 { + return nil, ftpError{err: fmt.Errorf(`failed parsing LIST entry: %s`, entry)} + } + + if skipSelfParent && (matches[3] == "." || matches[3] == "..") { + return nil, nil + } + + var err error + + var size uint64 + var mode os.FileMode = 0400 + if strings.Contains(matches[2], "DIR") { + mode |= os.ModeDir + } else { + size, err = strconv.ParseUint(matches[2], 10, 64) + if err != nil { + return nil, ftpError{err: fmt.Errorf(`failed parsing LIST entry's size: %s (%s)`, err, entry)} + } + } + + var mtime time.Time + for _, format := range dirTimeFormats { + if len(entry) >= len(format) { + mtime, err = time.ParseInLocation(format, matches[1], loc) + if err == nil { + break + } + } + } + + if err != nil { + return nil, ftpError{err: fmt.Errorf(`failed parsing LIST entry's mtime: %s (%s)`, err, entry)} + } + + info := &ftpFile{ + name: filepath.Base(matches[3]), + mode: mode, + mtime: mtime, + raw: entry, + size: int64(size), + } + + return info, nil +} + var lsRegex = regexp.MustCompile(`^\s*(\S)(\S{3})(\S{3})(\S{3})(?:\s+\S+){3}\s+(\d+)\s+(\w+\s+\d+)\s+([\d:]+)\s+(.+)$`) // total 404456 @@ -332,7 +392,8 @@ func parseLIST(entry string, loc *time.Location, skipSelfParent bool) (os.FileIn matches := lsRegex.FindStringSubmatch(entry) if len(matches) == 0 { - return nil, ftpError{err: fmt.Errorf(`failed parsing LIST entry: %s`, entry)} + info, err := parseDirLIST(entry, loc, skipSelfParent) + return info, err } if skipSelfParent && (matches[8] == "." || matches[8] == "..") { From 08121014527d4e755149a92f847c3663bf79a470 Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Mon, 12 Sep 2016 23:49:59 +0300 Subject: [PATCH 07/14] fix unexpected "426 Data connection closed..." * some servers send a 426 when the data connection fails during transfer * this response remains in the queue breaking all commands sent after it --- persistent_connection.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/persistent_connection.go b/persistent_connection.go index 7b0b4bc..2aa1b93 100644 --- a/persistent_connection.go +++ b/persistent_connection.go @@ -180,6 +180,17 @@ func (pconn *persistentConn) readResponse() (int, string, error) { err: fmt.Errorf("error reading response: %s", err), temporary: true, } + } else if code == replyConnectionClosed { + pconn.controlConn.SetReadDeadline(time.Now().Add(pconn.config.Timeout)) + code, msg, err = pconn.reader.ReadResponse(0) + if err != nil { + pconn.broken = true + pconn.debug("error reading second response: %s", err) + err = ftpError{ + err: fmt.Errorf("error reading response: %s", err), + temporary: true, + } + } } return code, msg, err } From 59f0223ec3e0e8a63a8c0dd49998b98ef0320a0d Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Tue, 13 Sep 2016 00:19:19 +0300 Subject: [PATCH 08/14] ignore incomplete MLSD / LIST entries without failing ReadDir * just write to the debug log --- file_system.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/file_system.go b/file_system.go index e22a7f2..bdba215 100644 --- a/file_system.go +++ b/file_system.go @@ -148,7 +148,9 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { info, err := parser(entry, true) if err != nil { c.debug("error in ReadDir: %s", err) - return nil, err + if !strings.Contains(err.Error(), "incomplete") { + return nil, err + } } if info == nil { From b1ca06d3394f28ad6b6c1d26d86d65a6ea70e399 Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Tue, 13 Sep 2016 00:47:36 +0300 Subject: [PATCH 09/14] remember if MLSD is not supported * no point in spamming it before every LIST * helps when accessing high latency servers --- file_system.go | 52 +++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/file_system.go b/file_system.go index bdba215..ab7c795 100644 --- a/file_system.go +++ b/file_system.go @@ -125,23 +125,34 @@ func commandNotSupporterdError(err error) bool { // be used. You may have to set ServerLocation in your config to get (more) // accurate ModTimes in this case. func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { - entries, err := c.dataStringList("MLSD %s", path) + pconn, err := c.getIdleConn() + if err != nil { + return nil, err + } + defer c.returnConn(pconn) + + var entries []string + + if !pconn.mlsdNotSupported { + entries, err = c.dataStringList(pconn, "MLSD %s", path) + } parser := parseMLST - if err != nil { - if !commandNotSupporterdError(err) { + if pconn.mlsdNotSupported || err != nil { + if !pconn.mlsdNotSupported && !commandNotSupporterdError(err) { return nil, err } + pconn.mlsdNotSupported = true - entries, err = c.dataStringList("LIST %s", path) + entries, err = c.dataStringList(pconn, "LIST %s", path) if err != nil { return nil, err } parser = func(entry string, skipSelfParent bool) (os.FileInfo, error) { return parseLIST(entry, c.config.ServerLocation, skipSelfParent) } - } + } var ret []os.FileInfo for _, entry := range entries { @@ -169,10 +180,17 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { // is a directory. You may have to set ServerLocation in your config to get // (more) accurate ModTimes when using "LIST". func (c *Client) Stat(path string) (os.FileInfo, error) { - lines, err := c.controlStringList("MLST %s", path) + pconn, err := c.getIdleConn() + if err != nil { + return nil, err + } + + defer c.returnConn(pconn) + + lines, err := c.controlStringList(pconn, "MLST %s", path) if err != nil { if commandNotSupporterdError(err) { - lines, err = c.dataStringList("LIST %s", path) + lines, err = c.dataStringList(pconn, "LIST %s", path) if err != nil { return nil, err } @@ -204,17 +222,10 @@ func extractDirName(msg string) (string, error) { return strings.Replace(msg[openQuote+1:closeQuote], `""`, `"`, -1), nil } -func (c *Client) controlStringList(f string, args ...interface{}) ([]string, error) { - pconn, err := c.getIdleConn() - if err != nil { - return nil, err - } - - defer c.returnConn(pconn) - +func (c *Client) controlStringList(pconn *persistentConn, f string, args ...interface{}) ([]string, error) { cmd := fmt.Sprintf(f, args...) - code, msg, err := pconn.sendCommand(cmd) + code, msg, _ := pconn.sendCommand(cmd) if !positiveCompletionReply(code) { pconn.debug("unexpected response to %s: %d-%s", cmd, code, msg) @@ -224,14 +235,7 @@ func (c *Client) controlStringList(f string, args ...interface{}) ([]string, err return strings.Split(msg, "\n"), nil } -func (c *Client) dataStringList(f string, args ...interface{}) ([]string, error) { - pconn, err := c.getIdleConn() - if err != nil { - return nil, err - } - - defer c.returnConn(pconn) - +func (c *Client) dataStringList(pconn *persistentConn, f string, args ...interface{}) ([]string, error) { dcGetter, err := pconn.prepareDataConn() if err != nil { return nil, err From dec280eada0141b21132c43fa1f74ceacf98722a Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Tue, 13 Sep 2016 01:04:46 +0300 Subject: [PATCH 10/14] add CWD command support (Setwd function) * some servers return "not found" with "MLST path" or "LIST path" * running "CWD path" and "LIST" or "MLST" works instead --- file_system.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/file_system.go b/file_system.go index ab7c795..1d07930 100644 --- a/file_system.go +++ b/file_system.go @@ -112,6 +112,27 @@ func (c *Client) Getwd() (string, error) { return dir, nil } +// Setwd changes the current working directory. +func (c *Client) Setwd(path string) (error) { + pconn, err := c.getIdleConn() + if err != nil { + return err + } + + defer c.returnConn(pconn) + + code, msg, err := pconn.sendCommand("CWD %s", path) + if err != nil { + return err + } + + if code != replyFileActionOkay { + return ftpError{code: code, msg: msg} + } + + return nil +} + func commandNotSupporterdError(err error) bool { respCode := err.(ftpError).Code() return respCode == replyCommandSyntaxError || respCode == replyCommandNotImplemented From 5e0f0e8db3e6a4aa4262ffbc64a97774d00ea2c2 Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Tue, 13 Sep 2016 11:40:03 +0300 Subject: [PATCH 11/14] fix tests - pureftp moved to https --- build_test_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_test_server.sh b/build_test_server.sh index b086940..f718353 100755 --- a/build_test_server.sh +++ b/build_test_server.sh @@ -84,7 +84,7 @@ VpOorURz8ETlfAA= -----END CERTIFICATE----- CERT -curl -O http://download.pureftpd.org/pub/pure-ftpd/releases/obsolete/pure-ftpd-1.0.36.tar.gz +curl -k -O https://download.pureftpd.org/pub/pure-ftpd/releases/obsolete/pure-ftpd-1.0.36.tar.gz tar -xzf pure-ftpd-1.0.36.tar.gz cd pure-ftpd-1.0.36 From abfd949ef365382f7c98f2021915cfca98c2980e Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Tue, 13 Sep 2016 11:58:17 +0300 Subject: [PATCH 12/14] clarify documentation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 32604ac..3ecb881 100644 --- a/README.md +++ b/README.md @@ -19,5 +19,6 @@ Pull requests or feature requests are welcome, but in the case of the former, yo ### Tests ### How to run tests (windows not supported): +* do not use the "root" account, use an unprivileged user with write access to the root goftp directory * ```./build_test_server.sh``` from root goftp directory (this downloads and compiles pure-ftpd and proftpd) * ```go test``` from the root goftp directory From 1ac9e7fd3f2225f8322f84ce37dc6afffac3a54b Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Tue, 13 Sep 2016 12:49:29 +0300 Subject: [PATCH 13/14] cleanup spaces --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index f4cd922..46a8c37 100644 --- a/client.go +++ b/client.go @@ -425,7 +425,7 @@ func (c *Client) openConn(idx int, host string) (pconn *persistentConn, err erro if pconn.hasFeature("UTF8") { if err = pconn.setUnicode(); err != nil { - goto Error + goto Error } } From abe3739e5540d6622d5bdbc77c0ba0baae9b080e Mon Sep 17 00:00:00 2001 From: rvicentiu Date: Tue, 13 Sep 2016 12:52:42 +0300 Subject: [PATCH 14/14] cleanup spaces --- file_system.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/file_system.go b/file_system.go index 1d07930..9cd3cae 100644 --- a/file_system.go +++ b/file_system.go @@ -173,7 +173,7 @@ func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { parser = func(entry string, skipSelfParent bool) (os.FileInfo, error) { return parseLIST(entry, c.config.ServerLocation, skipSelfParent) } - } + } var ret []os.FileInfo for _, entry := range entries {