diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index df7a174..6a4d1f9 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -2,7 +2,12 @@ name: Compile Test on: + pull_request: + branches: [ master ] + types: [ opened ] + push: + paths: [ '**.go' ] branches-ignore: - master @@ -15,7 +20,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.23.5 + go-version: 1.24.0 check-latest: true - name: Go Format diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index efb1bc1..3af8f0a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.23.5 + go-version: 1.24.0 check-latest: true - name: Checkout Repo diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 85619f1..b6a6ff5 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -2,6 +2,10 @@ name: Vulnerability Check on: + pull_request: + branches: [ master ] + types: [ opened ] + push: paths: [ '**.go' ] branches-ignore: @@ -16,7 +20,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.23.5 + go-version: 1.24.0 check-latest: true - name: Go Format diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a509ea9..fd8bf54 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,6 +2,10 @@ name: Go Tests on: + pull_request: + branches: [ master ] + types: [ opened ] + push: paths: [ '**.go' ] branches-ignore: @@ -16,7 +20,8 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.23.5 + go-version: 1.24.0 + check-latest: true - name: Go Format run: gofmt -s -w . && git diff --exit-code @@ -24,5 +29,11 @@ jobs: - name: Get Dependencies run: go get ./... - - name: Go Tests - run: go test -v -count=1 -race -shuffle=on -coverprofile=coverage.txt ./... \ No newline at end of file +# - name: Go Tests +# run: go test -v -count=1 -race -shuffle=on -coverprofile=coverage.txt ./... + + - name: Test ./internal/distlog + run: go test -v -count=1 -race -shuffle=on ./internal/distlog + + - name: Test ./internal/conf + run: go test -v -count=1 -race -shuffle=on ./internal/conf \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1ad1f50..57ebf83 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,6 @@ *.log scripts/ docs/test/* -playground/ +test/ terraform/ bin/* diff --git a/Makefile b/Makefile index 8cf109d..9694548 100644 --- a/Makefile +++ b/Makefile @@ -11,5 +11,6 @@ build: install run: build test: install build - @go test ./internal/conf - @./bin/mac/s3p use -f "docs/test/test_profile_aws.yaml" \ No newline at end of file + @go test -v ./internal/conf + @go test -v ./internal/distlog + @./bin/mac/s3p use -f "test/configs/aws.yaml" \ No newline at end of file diff --git a/README.md b/README.md index 2346fb6..67e82b7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# s3p - A configurable profile-based S3 backup and upload tool. +# s3p - A configurable profile-based object backup and upload tool. **CLI for Linux/MacOS** **supports** Amazon S3 **|** Google Cloud Storage **|** Linode (Akamai) Object Storage **|** Oracle Cloud Object Storage @@ -8,20 +8,26 @@ Oracle Cloud Object Storage [![Go Report Card][go_report_img]][go_report_url] [![Repo License][repo_license_img]][repo_license_url] -[![Code Quality](https://github.com/orme292/s3packer/actions/workflows/golang.yml/badge.svg)](https://github.com/orme292/s3packer/actions/workflows/golang.yml) +![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/orme292/s3packer/tests.yml?style=for-the-badge&label=Tests&labelColor=blue) +![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/orme292/s3packer/quality.yml?style=for-the-badge&label=Vulnerabilities&labelColor=blue) +![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/orme292/s3packer/compile.yml?style=for-the-badge&label=Compiles&labelColor=blue) + + + + --- [![Jetbrains_OSS][jetbrains_logo]][jetbrains_oss_url] [![Jetbrains_GoLand][jetbrains_goland_logo]][jetbrains_goland_url] -Special thanks to JetBrains! **s3p** is developed with help from JetBrains' Open Source program. +Special thanks to JetBrains! **s3p** was developed with help from JetBrains' Open Source program. --- ## About s3p is an S3 / Object Storage upload and backup tool. It uses YAML-based configs that tell it what to upload, where to upload, how to name, and how to tag the objects. s3p makes backup redundancy easier by using separate profiles -for buckets and providers. Currently it supports AWS, OCI (Oracle Cloud), and Linode (Akamai). +for buckets and providers. Currently, it supports AWS, Google Cloud, Linode (Akamai), OCI (Oracle Cloud). --- @@ -302,9 +308,11 @@ issue on [GitHub][issue_repo_url]. [releases_url]: https://github.com/orme292/s3packer/releases [issue_repo_url]: https://github.com/orme292/s3packer/issues/new/choose -[go_version_url]: https://golang.org/doc/go1.23 +[go_version_url]: https://golang.org/doc/go1.24 [go_report_url]: https://goreportcard.com/report/github.com/orme292/s3packer [repo_license_url]: https://github.com/orme292/s3packer/blob/master/LICENSE +[go_tests_url]: https://img.shields.io/github/actions/workflow/status/orme292/s3packer/tests.yml?style=for-the-badge&label=Tests + [s3_acl_url]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl [s3packer_aws_readme_url]: https://github.com/orme292/s3packer/blob/master/docs/using_aws.md diff --git a/go.mod b/go.mod index 33e9226..dc79b80 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module s3p -go 1.23.5 +go 1.24.0 require ( cloud.google.com/go/storage v1.50.0 @@ -16,6 +16,7 @@ require ( github.com/orme292/symwalker v0.2.2 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.10.0 golang.org/x/text v0.21.0 google.golang.org/api v0.214.0 gopkg.in/yaml.v3 v3.0.1 @@ -47,6 +48,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.3 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -62,9 +64,9 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.5.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect diff --git a/internal/distlog/log_level.go b/internal/distlog/log_level.go index e5a047f..2404b9a 100644 --- a/internal/distlog/log_level.go +++ b/internal/distlog/log_level.go @@ -36,6 +36,10 @@ func ParseLevel(n any) zerolog.Level { return WARN } + if lvl == zerolog.NoLevel { + return WARN + } + return lvl } diff --git a/internal/distlog/log_level_test.go b/internal/distlog/log_level_test.go new file mode 100644 index 0000000..25d1997 --- /dev/null +++ b/internal/distlog/log_level_test.go @@ -0,0 +1,74 @@ +package distlog + +import ( + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +func TestParseLevel(t *testing.T) { + tests := []struct { + name string + input any + expected zerolog.Level + }{ + { + name: "int 0 => debug", + input: 0, + expected: zerolog.DebugLevel, // "0" => debug + }, + { + name: "int 1 => info", + input: 1, + expected: zerolog.InfoLevel, // "1" => info + }, + { + name: "int 2 => warn", + input: 2, + expected: zerolog.WarnLevel, // "2" => warn + }, + { + name: "int 3 => error", + input: 3, + expected: zerolog.ErrorLevel, // "3" => error + }, + { + name: "bool true => warn (parses as '2')", + input: true, + expected: zerolog.WarnLevel, // "2" => warn + }, + { + name: "bool false => warn (parses as '2')", + input: false, + expected: zerolog.WarnLevel, // "2" => warn + }, + { + name: "any string => returns warn (code sets str = \"\" and fails parse)", + input: "", + expected: zerolog.WarnLevel, // because it never sets str=v, so parseLevel("") => error => default to warn + }, + { + name: "invalid type => warn (sets '2')", + input: struct{}{}, + expected: zerolog.WarnLevel, // "2" => warn + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ParseLevel(tt.input) + require.Equal(t, tt.expected, result, "ParseLevel(%v) did not return expected level", tt.input) + }) + } +} + +func TestZerologNumericLevels(t *testing.T) { + require.Equal(t, -1, int(zerolog.TraceLevel), "Trace level should be -1") + require.Equal(t, 0, int(zerolog.DebugLevel), "Debug level should be 0") + require.Equal(t, 1, int(zerolog.InfoLevel), "Info level should be 1") + require.Equal(t, 2, int(zerolog.WarnLevel), "Warn level should be 2") + require.Equal(t, 3, int(zerolog.ErrorLevel), "Error level should be 3") + require.Equal(t, 4, int(zerolog.FatalLevel), "Fatal level should be 4") + require.Equal(t, 5, int(zerolog.PanicLevel), "Panic level should be 5") +} diff --git a/internal/distlog/type_logbot.go b/internal/distlog/type_logbot.go index e78b196..4282f74 100644 --- a/internal/distlog/type_logbot.go +++ b/internal/distlog/type_logbot.go @@ -9,6 +9,12 @@ import ( "github.com/rs/zerolog/log" ) +const ( + EMPTY = "" + SPACE = " " + NEWLINE = "\n" +) + type LogOutput struct { Console bool File bool @@ -21,6 +27,10 @@ type LogBot struct { Logfile string } +// exitFunc is a function pointer that normally points to os.Exit. We override +// it in tests so we can verify calls to RouteLogMsg that would exit. +var exitFunc = os.Exit + func (lb *LogBot) SetLogLevel(lvl zerolog.Level) { zerolog.SetGlobalLevel(lvl) lb.Level = lvl @@ -35,7 +45,7 @@ func (lb *LogBot) BuildLogger(lvl zerolog.Level) zerolog.Logger { logFile, err := os.OpenFile(lb.Logfile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o640) if err != nil { log.Fatal().Msg("Unable to open log file.") - os.Exit(1) + exitFunc(1) } if lb.Output.Console { @@ -68,10 +78,11 @@ func (lb *LogBot) RouteLogMsg(lvl zerolog.Level, msg string) { z.WithLevel(lvl).Msg(msg) } - if lvl == zerolog.FatalLevel || lvl == zerolog.PanicLevel { - os.Exit(1) + if lvl == zerolog.FatalLevel { + exitFunc(1) + } else if lvl == zerolog.PanicLevel { + panic(fmt.Sprintf("unrecoverable error: %s", msg)) } - } /* @@ -79,51 +90,54 @@ Blast takes a string and passes it to LogBot.RouteLogMsg with zerolog.NoLevel. This ensures the message is logged regardless of the global log level. */ func (lb *LogBot) Blast(format string, a ...any) { - msg := fmt.Sprintf(format, a...) - lb.RouteLogMsg(BLAST, msg) + lb.RouteLogMsg(BLAST, getMsg(format, a...)) } func (lb *LogBot) Panic(format string, a ...any) { - msg := fmt.Sprintf(format, a...) - lb.RouteLogMsg(PANIC, msg) + lb.RouteLogMsg(PANIC, getMsg(format, a...)) } /* Fatal takes a string and passes it to LogBot.RouteLogMsg with zerolog.FatalLevel. */ func (lb *LogBot) Fatal(format string, a ...any) { - msg := fmt.Sprintf(format, a...) - lb.RouteLogMsg(FATAL, msg) + lb.RouteLogMsg(FATAL, getMsg(format, a...)) } /* Error takes a string and passes it to LogBot.RouteLogMsg with zerolog.ErrorLevel. */ func (lb *LogBot) Error(format string, a ...any) { - msg := fmt.Sprintf(format, a...) - lb.RouteLogMsg(ERROR, msg) + lb.RouteLogMsg(ERROR, getMsg(format, a...)) } /* Warn takes a string and passes it to LogBot.RouteLogMsg with zerolog.WarnLevel. */ func (lb *LogBot) Warn(format string, a ...any) { - msg := fmt.Sprintf(format, a...) - lb.RouteLogMsg(WARN, msg) + lb.RouteLogMsg(WARN, getMsg(format, a...)) } /* Info takes a string and passes it to LogBot.RouteLogMsg with zerolog.InfoLevel. */ func (lb *LogBot) Info(format string, a ...any) { - msg := fmt.Sprintf(format, a...) - lb.RouteLogMsg(INFO, msg) + lb.RouteLogMsg(INFO, getMsg(format, a...)) } /* Debug takes a string and passes it to LogBot.RouteLogMsg with zerolog.DebugLevel. */ func (lb *LogBot) Debug(format string, a ...any) { - msg := fmt.Sprintf(format, a...) - lb.RouteLogMsg(DEBUG, msg) + lb.RouteLogMsg(DEBUG, getMsg(format, a...)) +} + +func getMsg(format string, a ...any) string { + var msg string + if len(a) == 0 { + msg = format + } else { + msg = fmt.Sprintf(format, a...) + } + return msg } diff --git a/internal/distlog/type_logbot_test.go b/internal/distlog/type_logbot_test.go new file mode 100644 index 0000000..01c6ee5 --- /dev/null +++ b/internal/distlog/type_logbot_test.go @@ -0,0 +1,237 @@ +package distlog + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +var lb = &LogBot{ + Level: zerolog.DebugLevel, + Output: &LogOutput{ + Console: false, + File: false, + }, +} + +func restoreExitFunc() { + exitFunc = os.Exit +} + +// mockExit is a helper that replaces os.Exit. Instead of exiting the process, +// it will panic. This allows tests to catch the panic and confirm the exit call. +func mockExit(code int) { + panic(fmt.Sprintf("os.Exit called with code %d", code)) +} + +// captureOutput helps us redirect output to a buffer so we can inspect it. +func captureOutput(f func()) string { + oldStderr := os.Stderr + r, w, _ := os.Pipe() + os.Stderr = w + + f() + + w.Close() + os.Stderr = oldStderr + + var buf bytes.Buffer + _, _ = io.Copy(&buf, r) + return buf.String() +} + +func TestSetLogLevel(t *testing.T) { + lb.Level = zerolog.InfoLevel + lb.Output.Console = false + lb.Output.File = false + + require.Equal(t, zerolog.InfoLevel, lb.Level) + + lb.SetLogLevel(zerolog.DebugLevel) + require.Equal(t, zerolog.DebugLevel, lb.Level) + require.Equal(t, zerolog.GlobalLevel(), zerolog.DebugLevel) + + lb.SetLogLevel(zerolog.WarnLevel) + require.Equal(t, zerolog.WarnLevel, lb.Level) + require.Equal(t, zerolog.GlobalLevel(), zerolog.WarnLevel) +} + +// blConsoleOnly tests that the logger is built for console output only. +func blConsoleOnly(t *testing.T) { + lb.Output.Console = true + lb.Output.File = false + + // Capture console output + out := captureOutput(func() { + lb.SetLogLevel(zerolog.InfoLevel) + logger := lb.BuildLogger(zerolog.InfoLevel) + logger.Info().Msg("Testing console only") + }) + + require.Contains(t, out, "Testing console only", "Expected console output to contain log message.") +} + +// blFileOnly tests that the logger is built for file output only. +func blFileOnly(t *testing.T) { + tmpFile := filepath.Join(t.TempDir(), fmt.Sprintf("test-%d.log", time.Now().Unix())) + lb.Logfile = tmpFile + lb.Output.Console = false + lb.Output.File = true + + lb.SetLogLevel(zerolog.InfoLevel) + logger := lb.BuildLogger(zerolog.InfoLevel) + logger.Info().Msg("Testing file only logger") + + // Read the file and check content + data, err := os.ReadFile(tmpFile) + require.NoError(t, err) + require.Contains(t, string(data), "Testing file only logger", "Expected file output to contain log message.") +} + +// blConsoleAndFile tests that the logger is built for both console and file output. +func blConsoleAndFile(t *testing.T) { + tmpFile := filepath.Join(t.TempDir(), fmt.Sprintf("test-%d.log", time.Now().Unix())) + lb.Logfile = tmpFile + lb.Output.Console = true + lb.Output.File = true + + var consoleOut string + // Capture console output + consoleOut = captureOutput(func() { + lb.SetLogLevel(zerolog.InfoLevel) + logger := lb.BuildLogger(zerolog.InfoLevel) + logger.Info().Msg("Test console + file output") + }) + + data, err := os.ReadFile(tmpFile) + require.NoError(t, err) + + require.Contains(t, consoleOut, "Test console + file output", "Expected console output to contain log message.") + require.Contains(t, string(data), "Test console + file output", "Expected file output to contain log message.") +} + +func TestBuildLogger(t *testing.T) { + t.Run("BuildLogger-ConsoleOnly", blConsoleOnly) + t.Run("BuildLogger-FileOnly", blFileOnly) + t.Run("BuildLogger-ConsoleAndFile", blConsoleAndFile) +} + +// TestRouteLogMsg_Console tests that RouteLogMsg routes a message to console only. +func TestRouteLogMsg_Console(t *testing.T) { + lb.Output.Console = true + lb.Output.File = false + + msg := "route console test" + out := captureOutput(func() { + lb.SetLogLevel(zerolog.DebugLevel) + lb.RouteLogMsg(zerolog.DebugLevel, msg) + }) + + require.Contains(t, out, msg, "Expected console output to contain the routed message.") +} + +// TestRouteLogMsg_File tests that RouteLogMsg routes a message to file only. +func TestRouteLogMsg_File(t *testing.T) { + tmpFile := filepath.Join(t.TempDir(), fmt.Sprintf("test-%d.log", time.Now().Unix())) + lb.Logfile = tmpFile + lb.Output.Console = false + lb.Output.File = true + + msg := "route file test" + lb.SetLogLevel(zerolog.InfoLevel) + lb.RouteLogMsg(zerolog.InfoLevel, msg) + + data, err := os.ReadFile(tmpFile) + require.NoError(t, err) + require.Contains(t, string(data), msg, "Expected file output to contain the routed message.") +} + +// TestPanic verifies that calling Panic routes the message at panic level and calls os.Exit(1). +// We override exitFunc so the test does not terminate the process. +func TestPanic(t *testing.T) { + defer restoreExitFunc() + exitFunc = mockExit // override to catch the call + + lb := &LogBot{ + Level: zerolog.InfoLevel, + Output: &LogOutput{ + Console: true, + }, + } + + msg := "panic-level test" + + defer func() { + if r := recover(); r != nil { + log.Println(r.(string)) + require.True(t, strings.Contains(r.(string), msg), + "Expected os.Exit(1) to be called on Panic") + } else { + t.Errorf("Expected os.Exit(1) to be called on Panic but did not catch panic.") + } + }() + + lb.SetLogLevel(zerolog.PanicLevel) + lb.Panic("%s", msg) +} + +// TestFatal verifies that calling Fatal routes the message at fatal level and calls os.Exit(1). +func TestFatal(t *testing.T) { + defer restoreExitFunc() + exitFunc = mockExit + + lb := &LogBot{ + Level: zerolog.InfoLevel, + Output: &LogOutput{ + Console: true, + }, + } + + defer func() { + if r := recover(); r != nil { + require.True(t, strings.Contains(r.(string), "os.Exit called with code 1"), + "Expected os.Exit(1) to be called on Fatal") + } else { + t.Errorf("Expected os.Exit(1) to be called on Fatal but did not catch panic.") + } + }() + + lb.SetLogLevel(zerolog.FatalLevel) + lb.Fatal("fatal-level test") +} + +func TestMsg(t *testing.T) { + lb := &LogBot{ + Level: zerolog.DebugLevel, + Output: &LogOutput{ + Console: true, + }, + } + + levels := map[zerolog.Level]func(m string, a ...any){ + zerolog.DebugLevel: lb.Debug, + zerolog.InfoLevel: lb.Info, + zerolog.WarnLevel: lb.Warn, + zerolog.ErrorLevel: lb.Error, + zerolog.TraceLevel: lb.Blast, + } + for l, f := range levels { + lb.Level = l + lb.SetLogLevel(l) + msg := fmt.Sprintf("%s-level test", l.String()) + out := captureOutput(func() { + f("%s", msg) + }) + + require.Contains(t, out, msg, "Expected to log error message to console.") + } +} diff --git a/internal/distlog/type_logmsg.go b/internal/distlog/type_logmsg.go deleted file mode 100644 index 87ddc2f..0000000 --- a/internal/distlog/type_logmsg.go +++ /dev/null @@ -1,45 +0,0 @@ -package distlog - -import ( - "fmt" - - "github.com/rs/zerolog" -) - -const ( - EMPTY = "" - SPACE = " " - NEWLINE = "\n" -) - -type LogMsg struct { - Level zerolog.Level - LogMsg string - ScrMsg string - ScrIcon string -} - -func NewLogMsg(scrMsg, scrIcon string, level zerolog.Level, logMsg string) *LogMsg { - return &LogMsg{ - Level: level, - LogMsg: logMsg, - ScrMsg: scrMsg, - ScrIcon: scrIcon, - } -} - -func NewLogMsgB() *LogMsg { - return &LogMsg{} -} - -func (lm *LogMsg) setMessages(msg string) { - lm.ScrMsg = msg - lm.LogMsg = msg -} - -func (lm *LogMsg) SetMsgUpload(name string) *LogMsg { - lm.Level = INFO - lm.LogMsg = fmt.Sprintf("Uploading %s", name) - lm.ScrMsg = EMPTY - return lm -}