From 44e749e62e86e2eb80471497f8c78413478d4e39 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 25 Jun 2022 22:53:16 +0000 Subject: [PATCH 1/8] Start support for select column import and prefilter --- go.mod | 7 +- go.sum | 2 + main.go | 70 +++++++++++++++--- sql.go | 209 ++++++++++++++++++++++++++++++++++++++++++++++++++++ sql_test.go | 129 ++++++++++++++++++++++++++++++++ sqlite.go | 36 +++++++-- 6 files changed, 434 insertions(+), 19 deletions(-) create mode 100644 sql.go create mode 100644 sql_test.go diff --git a/go.mod b/go.mod index 5f4d685..19958f0 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,15 @@ module github.com/multiprocessio/dsq go 1.18 // Uncomment for local development (and re-run `go mod tidy`) -// replace github.com/multiprocessio/datastation/runner => ../datastation/runner +replace github.com/multiprocessio/datastation/runner => ../datastation/runner require ( github.com/chzyer/readline v1.5.0 github.com/google/uuid v1.3.0 github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862 github.com/olekukonko/tablewriter v0.0.5 + github.com/pganalyze/pg_query_go/v2 v2.1.0 + github.com/stretchr/testify v1.7.1 ) require ( @@ -37,6 +39,7 @@ require ( github.com/aws/smithy-go v1.9.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/denisenkom/go-mssqldb v0.12.0 // indirect github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect @@ -80,6 +83,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.34.0 // indirect @@ -114,4 +118,5 @@ require ( google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 02aed0f..4b4aad3 100644 --- a/go.sum +++ b/go.sum @@ -475,6 +475,8 @@ github.com/paulmach/orb v0.5.0 h1:sNhJV5ML+mv1F077ljOck/9inorF4ahDO8iNNpHbKHY= github.com/paulmach/orb v0.5.0/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pganalyze/pg_query_go/v2 v2.1.0 h1:donwPZ4G/X+kMs7j5eYtKjdziqyOLVp3pkUrzb9lDl8= +github.com/pganalyze/pg_query_go/v2 v2.1.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= diff --git a/main.go b/main.go index 48a3aeb..1d9569b 100644 --- a/main.go +++ b/main.go @@ -20,9 +20,9 @@ import ( "strings" "time" - "github.com/chzyer/readline" "github.com/multiprocessio/datastation/runner" + "github.com/chzyer/readline" "github.com/google/uuid" "github.com/olekukonko/tablewriter" ) @@ -343,6 +343,8 @@ type args struct { isInteractive bool convertNumbers bool noSQLiteWriter bool + noFieldsGuess bool + noPrefilter bool } func getArgs() (*args, error) { @@ -351,6 +353,8 @@ func getArgs() (*args, error) { args.noSQLiteWriter = strings.ToLower(os.Getenv("DSQ_NO_SQLITE_WRITER")) == "true" args.convertNumbers = strings.ToLower(os.Getenv("DSQ_CONVERT_NUMBERS")) == "true" args.cacheSettings.Enabled = strings.ToLower(os.Getenv("DSQ_CACHE")) == "true" + args.noFieldsGuess =strings.ToLower(os.Getenv("DSQ_NO_FIELDS_GUESS")) == "true" + args.noPrefilter =strings.ToLower(os.Getenv("DSQ_NO_PREFILTER")) == "true" osArgs := os.Args[1:] for i := 0; i < len(osArgs); i++ { @@ -427,11 +431,21 @@ func getArgs() (*args, error) { continue } - if arg == "--no-sqlite-writer" { + if arg == "-nsw" || arg == "--no-sqlite-writer" { args.noSQLiteWriter = true continue } + if arg == "-np" || arg == "--no-prefilter" { + args.noPrefilter = true + continue + } + + if arg == "-nfg" || arg == "--no-fields-guess" { + args.noFieldsGuess = true + continue + } + args.nonFlagArgs = append(args.nonFlagArgs, arg) } @@ -482,7 +496,7 @@ func _main() error { return nil } - lastNonFlagArg := "" + query := "" files := args.nonFlagArgs // Grab from stdin into local file @@ -507,8 +521,8 @@ func _main() error { // If -f|--file not present, query is the last argument if args.sqlFile == "" { if len(files) > 1 { - lastNonFlagArg = files[len(files)-1] - if strings.Contains(lastNonFlagArg, " ") { + query = files[len(files)-1] + if strings.Contains(query, " ") { files = files[:len(files)-1] } } @@ -519,8 +533,8 @@ func _main() error { return errors.New("Error opening sql file: " + err.Error()) } - lastNonFlagArg = string(content) - if lastNonFlagArg == "" { + query = string(content) + if query == "" { return errors.New("SQL file is empty.") } } @@ -582,7 +596,7 @@ func _main() error { connector.DatabaseConnectorInfo.Database.Database = cachedPath } - justDumpResults := lastNonFlagArg == "" && !args.isInteractive + justDumpResults := query == "" && !args.isInteractive // Check if we can use direct SQLite writer useSQLiteWriter := !args.noSQLiteWriter && !args.schema && !justDumpResults @@ -605,6 +619,7 @@ func _main() error { mtm := runner.MimeType(mt) useSQLiteWriter = useSQLiteWriter && (mtm == runner.CSVMimeType || mtm == runner.TSVMimeType || + mtm == runner.JSONLinesMimeType || mtm == runner.RegexpLinesMimeType) if !useSQLiteWriter { break @@ -621,8 +636,25 @@ func _main() error { } } + + var fieldsGuess []string + var prefilter func(map[string]any) bool + if query != "" && (!args.noFieldsGuess || !args.noPrefilter) { + a, ok := parse(rewriteQuery(query, &map[string]string{"0": "t_0"})) + if ok && !args.noFieldsGuess { + fieldsGuess, ok = identifiers(a) + if !ok { + fieldsGuess = nil + } + } + + if ok && !args.noPrefilter { + prefilter = filter(a) + } + } + // When dumping schema, need to injest even if cache is on. - if !args.cacheSettings.CachePresent || !args.cacheSettings.Enabled || lastNonFlagArg == "" { + if !args.cacheSettings.CachePresent || !args.cacheSettings.Enabled || query == "" { for i, file := range files { panelId := uuid.New().String() @@ -631,7 +663,14 @@ func _main() error { var w *runner.ResultWriter if useSQLiteWriter { tableName := fmt.Sprintf("t_%d", i) - sw, err := openSQLiteResultItemWriter(connector.DatabaseConnectorInfo.Database.Database, tableName, convertNumbers) + sw, err := openSQLiteResultItemWriter( + connector.DatabaseConnectorInfo.Database.Database, + tableName, + SQLiteResultItemWriterOptions{ + convertNumbers: convertNumbers, + prefilter: prefilter, + fieldsOverride: fieldsGuess, + }) if err != nil { return err } @@ -651,7 +690,14 @@ func _main() error { } } - panel, err := importFile(project.Id, panelId, file, mimetypeOverride[file], convertNumbers, w, !useSQLiteWriter) + panel, err := importFile( + project.Id, + panelId, + file, + mimetypeOverride[file], + convertNumbers, + w, + !useSQLiteWriter) if err != nil { return err } @@ -685,7 +731,7 @@ func _main() error { return repl(project, &ec, args, files, resolveDM_getPanelToId) } - return runQuery(lastNonFlagArg, project, &ec, args, files, resolveDM_getPanelToId) + return runQuery(query, project, &ec, args, files, resolveDM_getPanelToId) } func main() { diff --git a/sql.go b/sql.go new file mode 100644 index 0000000..7baee7d --- /dev/null +++ b/sql.go @@ -0,0 +1,209 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/multiprocessio/datastation/runner" + + q "github.com/pganalyze/pg_query_go/v2" +) + +func parse(query string) (*q.SelectStmt, bool) { + ast, err := q.Parse(query) + if err != nil { + return nil, false + } + + return ast.Stmts[0].Stmt.GetSelectStmt(), true +} + +func getValidIdentifier(n *q.Node) ([]string, bool) { + if n == nil { + return nil, false + } + + // Constants are fine + if n.GetAConst() != nil { + return nil, true + } + + if fc := n.GetFuncCall(); fc != nil { + var fields []string + for _, arg := range fc.Args { + _fields, ok := getValidIdentifier(arg) + if !ok { + return nil, false + } + + fields = append(fields, _fields...) + } + + return fields, true + } + + if e := n.GetAExpr(); e != nil { + l, ok := getValidIdentifier(e.Lexpr) + if !ok { + return nil, false + } + + r, ok := getValidIdentifier(e.Rexpr) + if !ok { + return nil, false + } + + return append(l, r...), true + } + + // Otherwise must be an identifier + cr := n.GetColumnRef() + if cr == nil { + return nil, false + } + + var parts []string + for _, field := range cr.Fields { + s := field.GetString_() + if s == nil { + return nil, false + } + + parts = append(parts, s.Str) + } + + s := strings.Join(parts, ".") + return []string{s}, true +} + +func identifiers(slct *q.SelectStmt) ([]string, bool) { + var fields []string + for _, t := range slct.TargetList { + v := t.GetResTarget().GetVal() + + _fields, ok := getValidIdentifier(v) + if !ok { + return nil, false + } + + fields = append(fields, _fields...) + } + + if len(slct.FromClause) != 1 { + return nil, false + } + rv := slct.FromClause[0].GetRangeVar() + if rv == nil { + return nil, false + } + if rv.GetRelname() == "" { + return nil, false + } + + if slct.WhereClause != nil { + where, ok := getValidIdentifier(slct.WhereClause) + if !ok { + return nil, false + } + fields = append(fields, where...) + } + + return fields, true +} + +func evalNode(n *q.Node, row map[string]any) any { + if n == nil { + return nil + } + + if c := n.GetAConst(); c != nil { + if i := c.GetVal().GetInteger(); i != nil { + return fmt.Sprintf("%d", i.Ival) + } + + if s := c.GetVal().GetString_(); s != nil { + return s.Str + } + + // Unsupported const type + return nil + } + + // Filtering on function calls unsupported + if fc := n.GetFuncCall(); fc != nil { + return nil + } + + if e := n.GetAExpr(); e != nil { + if len(e.Name) != 1 { + return nil + } + + _l := evalNode(e.Lexpr, row) + if _l == nil { + return nil + } + + l, ok := _l.(string) + if !ok { + return nil + } + + _r := evalNode(e.Rexpr, row) + if _r == nil { + return nil + } + + r, ok := _r.(string) + if !ok { + return nil + } + + switch e.Name[0].GetString_().Str { + case ">": + return l > r + case "<": + return l < r + case ">=": + return l >= r + case "<=": + return l <= r + case "=": + return l == r + } + } + + // Otherwise must be an identifier + cr := n.GetColumnRef() + if cr == nil { + return nil + } + + if len(cr.Fields) != 1 { + return nil + } + + s := cr.Fields[0].GetString_() + if s == nil { + return nil + } + + v := runner.GetObjectAtPath(row, s.Str) + switch v.(type) { + case string: + return v + default: + return fmt.Sprintf("%#v", v) + } +} + +func filter(slct *q.SelectStmt) func(m map[string]any) bool { + return func(row map[string]any) bool { + if slct.WhereClause == nil { + return false + } + + x := evalNode(slct.WhereClause, row) + return x != true && x != nil + } +} diff --git a/sql_test.go b/sql_test.go new file mode 100644 index 0000000..e5927f3 --- /dev/null +++ b/sql_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_identifiers(t *testing.T) { + tests := []struct { + in string + idents []string + ok bool + }{ + { + "SELECT a FROM x", + []string{"a"}, + true, + }, + { + "SELECT a, '1' FROM x", + []string{"a"}, + true, + }, + { + "SELECT a, b FROM x", + []string{"a", "b"}, + true, + }, + { + "SELECT avg(b) FROM x", + []string{"b"}, + true, + }, + { + "SELECT avg(b+1) FROM x", + []string{"b"}, + true, + }, + { + "SELECT b+1 FROM x", + []string{"b"}, + true, + }, + { + "SELECT a FROM x WHERE b > 1", + []string{"a", "b"}, + true, + }, + { + "SELECT * FROM x", + nil, + false, + }, + { + "SELECT a FROM x, y", + nil, + false, + }, + } + + for _, test := range tests { + t.Logf("Testing: %s", test.in) + s, err := parse(test.in) + assert.Nil(t, err) + + idents, ok := identifiers(s) + assert.Equal(t, test.idents, idents) + assert.Equal(t, test.ok, ok) + } +} + +func Test_filter(t *testing.T) { + tests := []struct { + query string + inRows []map[string]any + outRows []map[string]any + }{ + { + "SELECT a FROM x", + []map[string]any{ + {"a": 1}, + {"a": 2}, + }, + []map[string]any{ + {"a": 1}, + {"a": 2}, + }, + }, + { + "SELECT a FROM x WHERE avg(a) > 12", + []map[string]any{ + {"a": 1}, + {"a": 2}, + }, + []map[string]any{ + {"a": 1}, + {"a": 2}, + }, + }, + { + "SELECT a FROM x WHERE a = 12", + []map[string]any{ + {"a": 1}, + {"a": 12}, + }, + []map[string]any{ + {"a": 12}, + }, + }, + } + + for _, test := range tests { + t.Logf("Testing: %s", test.query) + s, err := parse(test.query) + assert.Nil(t, err) + + f := filter(s) + var end []map[string]any + for _, r := range test.inRows { + canFilter := f(r) + if !canFilter { + end = append(end, r) + } + } + + assert.Equal(t, test.outRows, end) + } +} diff --git a/sqlite.go b/sqlite.go index 5d74445..c9402e8 100644 --- a/sqlite.go +++ b/sqlite.go @@ -9,18 +9,28 @@ import ( "github.com/multiprocessio/datastation/runner" ) +type SQLiteResultItemWriterOptions struct { + convertNumbers bool + prefilter func(map[string]any) bool + fieldsOverride []string +} + type SQLiteResultItemWriter struct { + tableCreated bool db *sql.DB fields []string panelId string rowBuffer runner.Vector[any] - convertNumbers bool + + SQLiteResultItemWriterOptions } -func openSQLiteResultItemWriter(f string, panelId string, convertNumbers bool) (runner.ResultItemWriter, error) { +func openSQLiteResultItemWriter(f string, panelId string, opts SQLiteResultItemWriterOptions) (runner.ResultItemWriter, error) { var sw SQLiteResultItemWriter sw.panelId = panelId - sw.convertNumbers = convertNumbers + sw.SQLiteResultItemWriterOptions = opts + + sw.fields = opts.fieldsOverride sw.rowBuffer = runner.Vector[any]{} @@ -127,15 +137,26 @@ func (sw *SQLiteResultItemWriter) WriteRow(r any, written int) error { return fmt.Errorf("Row must be a map, got: %#v", r) } - if len(sw.fields) == 0 { - for key := range m { - sw.fields = append(sw.fields, key) + if sw.prefilter != nil { + canSkip := sw.prefilter(m) + if canSkip { + return nil + } + } + + if !sw.tableCreated { + if len(sw.fields) == 0 { + for key := range m { + sw.fields = append(sw.fields, key) + } } err := sw.createTable() if err != nil { return err } + + sw.tableCreated = true } for _, field := range sw.fields { @@ -147,6 +168,9 @@ func (sw *SQLiteResultItemWriter) WriteRow(r any, written int) error { return err } v = string(bs) + // TODO: don't keep this + case map[string]any: + v = nil } sw.rowBuffer.Append(v) } From eb12021d46f50323071eb072a01316e22bd8891f Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 25 Jun 2022 22:53:21 +0000 Subject: [PATCH 2/8] gofmt --- main.go | 11 +++++------ sqlite.go | 10 +++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index 1d9569b..70fddaa 100644 --- a/main.go +++ b/main.go @@ -343,8 +343,8 @@ type args struct { isInteractive bool convertNumbers bool noSQLiteWriter bool - noFieldsGuess bool - noPrefilter bool + noFieldsGuess bool + noPrefilter bool } func getArgs() (*args, error) { @@ -353,8 +353,8 @@ func getArgs() (*args, error) { args.noSQLiteWriter = strings.ToLower(os.Getenv("DSQ_NO_SQLITE_WRITER")) == "true" args.convertNumbers = strings.ToLower(os.Getenv("DSQ_CONVERT_NUMBERS")) == "true" args.cacheSettings.Enabled = strings.ToLower(os.Getenv("DSQ_CACHE")) == "true" - args.noFieldsGuess =strings.ToLower(os.Getenv("DSQ_NO_FIELDS_GUESS")) == "true" - args.noPrefilter =strings.ToLower(os.Getenv("DSQ_NO_PREFILTER")) == "true" + args.noFieldsGuess = strings.ToLower(os.Getenv("DSQ_NO_FIELDS_GUESS")) == "true" + args.noPrefilter = strings.ToLower(os.Getenv("DSQ_NO_PREFILTER")) == "true" osArgs := os.Args[1:] for i := 0; i < len(osArgs); i++ { @@ -636,7 +636,6 @@ func _main() error { } } - var fieldsGuess []string var prefilter func(map[string]any) bool if query != "" && (!args.noFieldsGuess || !args.noPrefilter) { @@ -668,7 +667,7 @@ func _main() error { tableName, SQLiteResultItemWriterOptions{ convertNumbers: convertNumbers, - prefilter: prefilter, + prefilter: prefilter, fieldsOverride: fieldsGuess, }) if err != nil { diff --git a/sqlite.go b/sqlite.go index c9402e8..d686962 100644 --- a/sqlite.go +++ b/sqlite.go @@ -11,16 +11,16 @@ import ( type SQLiteResultItemWriterOptions struct { convertNumbers bool - prefilter func(map[string]any) bool + prefilter func(map[string]any) bool fieldsOverride []string } type SQLiteResultItemWriter struct { tableCreated bool - db *sql.DB - fields []string - panelId string - rowBuffer runner.Vector[any] + db *sql.DB + fields []string + panelId string + rowBuffer runner.Vector[any] SQLiteResultItemWriterOptions } From e1b3fc91e74ac0637a5657f93ac51bcc840f5be2 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Mon, 27 Jun 2022 18:31:37 -0400 Subject: [PATCH 3/8] Update mod --- go.mod | 6 +++--- go.sum | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 19958f0..f6ab275 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/multiprocessio/dsq go 1.18 // Uncomment for local development (and re-run `go mod tidy`) -replace github.com/multiprocessio/datastation/runner => ../datastation/runner +// replace github.com/multiprocessio/datastation/runner => ../datastation/runner require ( github.com/chzyer/readline v1.5.0 github.com/google/uuid v1.3.0 - github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862 + github.com/multiprocessio/datastation/runner v0.0.0-20220627185241-7ae4dcfc07a8 github.com/olekukonko/tablewriter v0.0.5 github.com/pganalyze/pg_query_go/v2 v2.1.0 github.com/stretchr/testify v1.7.1 @@ -118,5 +118,5 @@ require ( google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4b4aad3..37bffe8 100644 --- a/go.sum +++ b/go.sum @@ -445,8 +445,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862 h1:bgzazkNqaktd7lQqOcG6TXG1iz8S1GeGbwEcy0/Zh9k= -github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862/go.mod h1:UCms/xK08DspNqDDZ5XsaIlc39AuREmeELspFYghMGI= +github.com/multiprocessio/datastation/runner v0.0.0-20220627185241-7ae4dcfc07a8 h1:SPWxV18blS+QYi/kDUh22DYw/g8f9kA2qNq1hUn9mVw= +github.com/multiprocessio/datastation/runner v0.0.0-20220627185241-7ae4dcfc07a8/go.mod h1:kfhv+2d/MANHKVAEkzw2aCkPsmUxuRpfy5Fz3jGpVgg= github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e h1:NlPl7amllnQyVAkZgjBvFEkKxJSba/R8ZpaTodc7SIQ= github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e/go.mod h1:huI4M/MrI5px/SgmXYi0a2byKikSLgDrnMQuXOqKtw4= github.com/multiprocessio/go-openoffice v0.0.0-20220110232726-064f5dda1956 h1:WVofL03Eq+z3LbDOfH5eKzu2U85LFZZngOMBlNaO/H0= @@ -1094,8 +1094,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From f6fd7c3a69f0d00cf63eafbadfc4d21a3efff182 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Tue, 28 Jun 2022 00:26:08 +0000 Subject: [PATCH 4/8] Fix for test --- sql_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql_test.go b/sql_test.go index e5927f3..f6a6441 100644 --- a/sql_test.go +++ b/sql_test.go @@ -61,8 +61,8 @@ func Test_identifiers(t *testing.T) { for _, test := range tests { t.Logf("Testing: %s", test.in) - s, err := parse(test.in) - assert.Nil(t, err) + s, ok := parse(test.in) + assert.True(t, ok) idents, ok := identifiers(s) assert.Equal(t, test.idents, idents) @@ -112,8 +112,8 @@ func Test_filter(t *testing.T) { for _, test := range tests { t.Logf("Testing: %s", test.query) - s, err := parse(test.query) - assert.Nil(t, err) + s, ok := parse(test.query) + assert.True(t, ok) f := filter(s) var end []map[string]any From 0e43f1b17a107aa50a8c9b01ff4b004d3f72616d Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Tue, 28 Jun 2022 00:43:30 +0000 Subject: [PATCH 5/8] Fix test --- scripts/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test.py b/scripts/test.py index 1b3f29c..8aaf0e3 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -321,7 +321,7 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon test("URL functions", to_run, want=want, sort=True) # URL functions, split_part -to_run = """./dsq testdata/basic_logs.csv 'SELECT split_part(url_host(request), ".", -1) host, count(1) count FROM {} group by host' """ +to_run = """./dsq testdata/basic_logs.csv "SELECT split_part(url_host(request), '.', -1) host, count(1) count FROM {} group by host" """ want = '[{"host":"com","count":2}]' test("URL functions", to_run, want=want, sort=True) From e1041730d3394bf3d97a6597ccdf6ee36a7d33ae Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Mon, 18 Jul 2022 01:47:36 +0000 Subject: [PATCH 6/8] Disable build on windows --- sql.go | 2 ++ sql_windows.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 sql_windows.go diff --git a/sql.go b/sql.go index 7baee7d..7de9c24 100644 --- a/sql.go +++ b/sql.go @@ -1,3 +1,5 @@ +//go:build !windows + package main import ( diff --git a/sql_windows.go b/sql_windows.go new file mode 100644 index 0000000..d47bb93 --- /dev/null +++ b/sql_windows.go @@ -0,0 +1,15 @@ +package main + +func parse(query string) (*q.SelectStmt, bool) { + return nil, false +} + +func identifiers(slct *q.SelectStmt) ([]string, bool) { + return nil, false +} + +func filter(slct *q.SelectStmt) func(m map[string]any) bool { + return func(_ map[string]any) bool { + return false + } +} From cd30379cbde815b77867c1f8a1e88e6e76c18fff Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Mon, 18 Jul 2022 02:09:25 +0000 Subject: [PATCH 7/8] Fix for windows --- main.go | 3 +++ sql_windows.go | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 228da17..8fe0cd7 100644 --- a/main.go +++ b/main.go @@ -632,6 +632,7 @@ func _main() error { var fieldsGuess []string var prefilter func(map[string]any) bool if query != "" && (!args.noFieldsGuess || !args.noPrefilter) { + fmt.Println("HERE?") a, ok := parse(rewriteQuery(query, &map[string]string{"0": "t_0"})) if ok && !args.noFieldsGuess { fieldsGuess, ok = identifiers(a) @@ -640,7 +641,9 @@ func _main() error { } } + fmt.Println("or here??") if ok && !args.noPrefilter { + fmt.Println("here?") prefilter = filter(a) } } diff --git a/sql_windows.go b/sql_windows.go index d47bb93..648b956 100644 --- a/sql_windows.go +++ b/sql_windows.go @@ -1,14 +1,16 @@ package main -func parse(query string) (*q.SelectStmt, bool) { +type SelectStmt struct{} + +func parse(query string) (*SelectStmt, bool) { return nil, false } -func identifiers(slct *q.SelectStmt) ([]string, bool) { +func identifiers(slct *SelectStmt) ([]string, bool) { return nil, false } -func filter(slct *q.SelectStmt) func(m map[string]any) bool { +func filter(slct *SelectStmt) func(m map[string]any) bool { return func(_ map[string]any) bool { return false } From 7187b40902a3285fb278112262243d428f436dd7 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Mon, 18 Jul 2022 20:00:40 +0000 Subject: [PATCH 8/8] Fix for floats --- main.go | 3 --- sql.go | 6 +++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 8fe0cd7..228da17 100644 --- a/main.go +++ b/main.go @@ -632,7 +632,6 @@ func _main() error { var fieldsGuess []string var prefilter func(map[string]any) bool if query != "" && (!args.noFieldsGuess || !args.noPrefilter) { - fmt.Println("HERE?") a, ok := parse(rewriteQuery(query, &map[string]string{"0": "t_0"})) if ok && !args.noFieldsGuess { fieldsGuess, ok = identifiers(a) @@ -641,9 +640,7 @@ func _main() error { } } - fmt.Println("or here??") if ok && !args.noPrefilter { - fmt.Println("here?") prefilter = filter(a) } } diff --git a/sql.go b/sql.go index 7de9c24..91322fb 100644 --- a/sql.go +++ b/sql.go @@ -120,7 +120,11 @@ func evalNode(n *q.Node, row map[string]any) any { if c := n.GetAConst(); c != nil { if i := c.GetVal().GetInteger(); i != nil { - return fmt.Sprintf("%d", i.Ival) + return i.String() + } + + if f := c.GetVal().GetFloat(); f != nil { + return f.GetStr() } if s := c.GetVal().GetString_(); s != nil {