diff --git a/hugo2confluence/.gitignore b/hugo2confluence/.gitignore index 2b0c6e44..4103f01a 100644 --- a/hugo2confluence/.gitignore +++ b/hugo2confluence/.gitignore @@ -25,3 +25,6 @@ go.work.sum .env bin/ + +# Avoid accidentally checking in Hugo content +content/ diff --git a/hugo2confluence/README.md b/hugo2confluence/README.md index 055f3ff0..c4c8d4bf 100644 --- a/hugo2confluence/README.md +++ b/hugo2confluence/README.md @@ -64,28 +64,35 @@ jobs: CONFLUENCE_SPACE: "Myspace" CONFLUENCE_ANCESTOR: "12345678" CONFLUENCE_ROOT: "content/" + CONFLUENCE_LABEL: "my-unique-label" CONFLUENCE_ORIG: "https://mywiki.example.com/" CONFLUENCE_EDIT: "https://github.com/myorg/mywiki/edit/main/content/" ``` -## Cleanup +## Ultities + +This section describes some utility programs in this folder +that may be needed in one-off situations. + +Note: These may possibly lead to `429 Too Many Requests` errors depending on size of the Hugo site. Wait a bit and try again. + +### Cleanup Utility If you make a mistake running this tool, you can delete all the pages created by it using the [cleanup](./cleanup/) tool: -```sh -export CONFLUENCE_DISPLAY="My Fullname" -export CONFLUENCE_USER="myemail@example.com" -export CONFLUENCE_TOKEN="..." -``` - ``` go run cleanup/cleanup.go ``` -This may possibly lead to `429 Too Many Requests` errors depending on size of the Hugo site. Wait a bit and try again. +### Label Utility + +If you need to retroactively apply a label to all the pages, +you can use the [label](./label/) tool: -Note: `CONFLUENCE_DISPLAY` is the user's display name (we are strangely unable to use `CONFLUENCE_USER` for search). +``` +go run label/label.go +``` ## TODO diff --git a/hugo2confluence/action.yaml b/hugo2confluence/action.yaml index f140e23b..3af700b5 100644 --- a/hugo2confluence/action.yaml +++ b/hugo2confluence/action.yaml @@ -16,6 +16,7 @@ runs: "CONFLUENCE_SPACE" "CONFLUENCE_ANCESTOR" "CONFLUENCE_ROOT" + "CONFLUENCE_LABEL" ) for v in ${required_env_vars[@]}; do if [[ "${!v}" == "" ]]; then diff --git a/hugo2confluence/cleanup/cleanup.go b/hugo2confluence/cleanup/cleanup.go index e25b2aba..f3340119 100644 --- a/hugo2confluence/cleanup/cleanup.go +++ b/hugo2confluence/cleanup/cleanup.go @@ -12,10 +12,10 @@ import ( var env = envconfig.MustProcess(context.Background(), &struct { ConfluenceUser string `env:"CONFLUENCE_USER,required"` ConfluenceToken string `env:"CONFLUENCE_TOKEN,required"` - ConfluenceDisplay string `env:"CONFLUENCE_DISPLAY,required"` ConfluenceURL string `env:"CONFLUENCE_URL,required"` ConfluenceSpace string `env:"CONFLUENCE_SPACE,required"` ConfluenceAncestor string `env:"CONFLUENCE_ANCESTOR,required"` + ConfluenceLabel string `env:"CONFLUENCE_LABEL,required"` }{}) func main() { @@ -26,7 +26,7 @@ func main() { for { resp, err := api.Search(confluence.SearchQuery{ - CQL: fmt.Sprintf(`space="%s"`, env.ConfluenceSpace), + CQL: fmt.Sprintf(`space=%q and label=%q`, env.ConfluenceSpace, env.ConfluenceLabel), Expand: []string{"content.history,content.ancestors"}, Limit: 100, }) @@ -40,10 +40,6 @@ func main() { if result.Content.History == nil || result.Content.Ancestors == nil { continue } - /// Safe guard from deleting items not created by the intended user - if result.Content.History.CreatedBy.DisplayName != env.ConfluenceDisplay { - continue - } // Now only delete if this page is in the lineage of the root ancestor for _, ancestor := range result.Content.Ancestors { diff --git a/hugo2confluence/go.mod b/hugo2confluence/go.mod index e9f11b6e..caacef1a 100644 --- a/hugo2confluence/go.mod +++ b/hugo2confluence/go.mod @@ -7,4 +7,8 @@ require ( github.com/virtomize/confluence-go-api v1.5.0 ) -require github.com/magefile/mage v1.14.0 // indirect +require github.com/magefile/mage v1.15.0 // indirect + +// This is needed in order to use ther SearchWithNext method +// See https://github.com/Virtomize/confluence-go-api/pull/69 +replace github.com/virtomize/confluence-go-api => github.com/jdolitsky/confluence-go-api v0.0.0-20250401174407-48bdb8a37784 diff --git a/hugo2confluence/go.sum b/hugo2confluence/go.sum index a3d97e0c..058721db 100644 --- a/hugo2confluence/go.sum +++ b/hugo2confluence/go.sum @@ -2,8 +2,11 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= +github.com/jdolitsky/confluence-go-api v0.0.0-20250401174407-48bdb8a37784 h1:WfMCvZoFr3myJ/W6ymiJmbNNzfBTwHJn4tjI0P7Vj5c= +github.com/jdolitsky/confluence-go-api v0.0.0-20250401174407-48bdb8a37784/go.mod h1:a96WPcok5g+7l5LC/ztcrp4cLmrIA1DHxxZSv/iqvsQ= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sethvargo/go-envconfig v1.1.1 h1:JDu8Q9baIzJf47NPkzhIB6aLYL0vQ+pPypoYrejS9QY= @@ -11,5 +14,3 @@ github.com/sethvargo/go-envconfig v1.1.1/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/virtomize/confluence-go-api v1.5.0 h1:CRdlL6V/78szvTjZA6XBSevsmqxDkqbDkkrmyoLpa9Y= -github.com/virtomize/confluence-go-api v1.5.0/go.mod h1:a96WPcok5g+7l5LC/ztcrp4cLmrIA1DHxxZSv/iqvsQ= diff --git a/hugo2confluence/label/label.go b/hugo2confluence/label/label.go new file mode 100644 index 00000000..0c611241 --- /dev/null +++ b/hugo2confluence/label/label.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/sethvargo/go-envconfig" + confluence "github.com/virtomize/confluence-go-api" +) + +var env = envconfig.MustProcess(context.Background(), &struct { + ConfluenceUser string `env:"CONFLUENCE_USER,required"` + ConfluenceToken string `env:"CONFLUENCE_TOKEN,required"` + ConfluenceURL string `env:"CONFLUENCE_URL,required"` + ConfluenceSpace string `env:"CONFLUENCE_SPACE,required"` + ConfluenceAncestor string `env:"CONFLUENCE_ANCESTOR,required"` + ConfluenceLabel string `env:"CONFLUENCE_LABEL,required"` +}{}) + +func main() { + api, err := confluence.NewAPI(env.ConfluenceURL, env.ConfluenceUser, env.ConfluenceToken) + if err != nil { + log.Fatal(err) + } + + next := "" + query := confluence.SearchQuery{ + CQL: fmt.Sprintf(`space=%q and label in (%q)`, env.ConfluenceSpace, env.ConfluenceLabel), + Expand: []string{"content.history,content.ancestors"}, + Limit: 100, + } + + for { + resp, err := api.SearchWithNext(query, next) + if err != nil { + log.Fatal(err) + } + if resp == nil { + break + } + for _, result := range resp.Results { + if result.Content.History == nil || result.Content.Ancestors == nil { + continue + } + + // Now only delete if this page is in the lineage of the root ancestor + for _, ancestor := range result.Content.Ancestors { + if ancestor.ID == env.ConfluenceAncestor { + id := result.Content.ID + // Need to apply the label for cleanup purposes later + log.Printf("Applying label \"%s\" to \"%s\" (id:%s)", + env.ConfluenceLabel, result.Content.Title, id) + labels := []confluence.Label{{Name: env.ConfluenceLabel}} + if _, err := api.AddLabels(id, &labels); err != nil { + log.Printf("unable to label item %s: %v", id, err) + } + break + } + } + } + next = resp.Links.Next + if next == "" { + break + } + log.Printf("Using next page: %s", next) + } +} diff --git a/hugo2confluence/main.go b/hugo2confluence/main.go index c0d9787a..e18c0e3a 100644 --- a/hugo2confluence/main.go +++ b/hugo2confluence/main.go @@ -11,7 +11,9 @@ import ( "net/http" "os" "path/filepath" + "slices" "strings" + "time" "github.com/sethvargo/go-envconfig" confluence "github.com/virtomize/confluence-go-api" @@ -24,6 +26,7 @@ var env = envconfig.MustProcess(context.Background(), &struct { ConfluenceSpace string `env:"CONFLUENCE_SPACE,required"` ConfluenceAncestor string `env:"CONFLUENCE_ANCESTOR,required"` ConfluenceRoot string `env:"CONFLUENCE_ROOT,required"` + ConfluenceLabel string `env:"CONFLUENCE_LABEL,required"` ConfluenceOrig string `env:"CONFLUENCE_ORIG"` ConfluenceEdit string `env:"CONFLUENCE_EDIT"` }{}) @@ -47,7 +50,14 @@ func main() { log.Printf("Starting site sync to Confluence (url: %s space:%s ancestor:%s)", env.ConfluenceURL, env.ConfluenceSpace, env.ConfluenceAncestor) - if err := syncPage(api, rootPage); err != nil { + ids, err := syncPage(api, rootPage) + if err != nil { + log.Fatal(err) + } + + log.Printf("Cleaning up old Confluence pages (url: %s space:%s ancestor:%s)", + env.ConfluenceURL, env.ConfluenceSpace, env.ConfluenceAncestor) + if err := cleanupPages(api, ids); err != nil { log.Fatal(err) } @@ -106,13 +116,13 @@ func getPages(rootDir string) ([]page, error) { return pages, nil } -func syncPage(api *confluence.API, p page) error { +func syncPage(api *confluence.API, p page) ([]string, error) { log.Printf("Syncing %s", p.filePath) // Generate the content content, err := contentFromFile(p.parentID, p.filePath) if err != nil { - return err + return nil, err } // Check if the page already exists to determine correct version @@ -135,7 +145,7 @@ func syncPage(api *confluence.API, p page) error { Title: t, }) if err != nil { - return err + return nil, err } if len(resp.Results) == 0 { break @@ -172,21 +182,86 @@ func syncPage(api *confluence.API, p page) error { } if writeErr != nil { - return writeErr + return nil, writeErr } log.Printf("Wrote page \"%s\" (parent:%s id:%s version:%d url:%s)", written.Title, p.parentID, written.ID, written.Version.Number, written.Links.WebUI) + // Need to apply the label for cleanup purposes later + log.Printf("Applying label \"%s\" to \"%s\" (parent:%s id:%s)", + env.ConfluenceLabel, written.Title, p.parentID, written.ID) + labels := []confluence.Label{{Name: env.ConfluenceLabel}} + if _, err := api.AddLabels(written.ID, &labels); err != nil { + + // Wait 10 seconds and retry (sometimes this endpoint returns a 500?) + log.Printf("Got error applying label, retrying in 10 seconds: %s", err) + time.Sleep(time.Second * 10) + if _, err := api.AddLabels(written.ID, &labels); err != nil { + return nil, err + } + } + + // List of all IDs written in this function call, including recursive ones + ids := []string{written.ID} + // Now recursively write the children // TODO: sync children pages asynchronously? for _, child := range p.children { child.parentID = written.ID - if err := syncPage(api, child); err != nil { - return err + childIds, err := syncPage(api, child) + if err != nil { + return nil, err } + ids = append(childIds, ids...) } + return ids, nil +} + +func cleanupPages(api *confluence.API, exceptIds []string) error { + query := confluence.SearchQuery{ + CQL: fmt.Sprintf(`space=%q and label=%q`, env.ConfluenceSpace, env.ConfluenceLabel), + Expand: []string{"content.history,content.ancestors"}, + Limit: 100, + } + next := "" + for { + resp, err := api.SearchWithNext(query, next) + if err != nil { + log.Fatal(err) + } + if resp == nil { + break + } + for _, result := range resp.Results { + if result.Content.History == nil || result.Content.Ancestors == nil { + continue + } + + // If the ID is one of the ones we have written as part of this run, ignore it + id := result.Content.ID + if slices.Contains(exceptIds, id) { + continue + } + + // Now only delete if this page is in the lineage of the root ancestor + for _, ancestor := range result.Content.Ancestors { + if ancestor.ID == env.ConfluenceAncestor { + log.Printf("Cleaning up page %q (id: %s)", result.Content.Title, id) + /*if _, err := api.DelContent(id); err != nil { + return fmt.Errorf("unable to delete item %s: %v", id, err) + }*/ + break + } + } + } + next = resp.Links.Next + if next == "" { + break + } + log.Printf("Using next page: %s", next) + } return nil }