hugo2confluence: auto cleanup pages removed in hugo

hugo2confluence/.gitignore

@@ -25,3 +25,6 @@ go.work.sum
 .env
 
 bin/
+
+# Avoid accidentally checking in Hugo content
+content/

hugo2confluence/README.md

@@ -64,28 +64,35 @@ jobs:
           CONFLUENCE_SPACE: "Myspace"
           CONFLUENCE_ANCESTOR: "12345678"
           CONFLUENCE_ROOT: "content/"
+          CONFLUENCE_LABEL: "my-unique-label"
           CONFLUENCE_ORIG: "https://mywiki.example.com/"
           CONFLUENCE_EDIT: "https://github.com/myorg/mywiki/edit/main/content/"
 ```
 
-## Cleanup
+## Ultities
+
+This section describes some utility programs in this folder
+that may be needed in one-off situations.
+
+Note: These may possibly lead to `429 Too Many Requests` errors depending on size of the Hugo site. Wait a bit and try again.
+
+### Cleanup Utility
 
 If you make a mistake running this tool, you can delete all the pages
 created by it using the [cleanup](./cleanup/) tool:
 
-```sh
-export CONFLUENCE_DISPLAY="My Fullname"
-export CONFLUENCE_USER="myemail@example.com"
-export CONFLUENCE_TOKEN="..."
-```
-
 ```
 go run cleanup/cleanup.go
 ```
 
-This may possibly lead to `429 Too Many Requests` errors depending on size of the Hugo site. Wait a bit and try again.
+### Label Utility
+
+If you need to retroactively apply a label to all the pages,
+you can use the [label](./label/) tool:
 
-Note: `CONFLUENCE_DISPLAY` is the user's display name (we are strangely unable to use `CONFLUENCE_USER` for search).
+```
+go run label/label.go
+```
 
 ## TODO
 

hugo2confluence/action.yaml

@@ -16,6 +16,7 @@ runs:
           "CONFLUENCE_SPACE"
           "CONFLUENCE_ANCESTOR"
           "CONFLUENCE_ROOT"
+          "CONFLUENCE_LABEL"
         )
         for v in ${required_env_vars[@]}; do
           if [[ "${!v}" == "" ]]; then

hugo2confluence/cleanup/cleanup.go

@@ -12,10 +12,10 @@ import (
 var env = envconfig.MustProcess(context.Background(), &struct {
 	ConfluenceUser     string `env:"CONFLUENCE_USER,required"`
 	ConfluenceToken    string `env:"CONFLUENCE_TOKEN,required"`
-	ConfluenceDisplay  string `env:"CONFLUENCE_DISPLAY,required"`
 	ConfluenceURL      string `env:"CONFLUENCE_URL,required"`
 	ConfluenceSpace    string `env:"CONFLUENCE_SPACE,required"`
 	ConfluenceAncestor string `env:"CONFLUENCE_ANCESTOR,required"`
+	ConfluenceLabel    string `env:"CONFLUENCE_LABEL,required"`
 }{})
 
 func main() {
@@ -26,7 +26,7 @@ func main() {
 
 	for {
 		resp, err := api.Search(confluence.SearchQuery{
-			CQL:    fmt.Sprintf(`space="%s"`, env.ConfluenceSpace),
+			CQL:    fmt.Sprintf(`space=%q and label=%q`, env.ConfluenceSpace, env.ConfluenceLabel),
 			Expand: []string{"content.history,content.ancestors"},
 			Limit:  100,
 		})
@@ -40,10 +40,6 @@ func main() {
 			if result.Content.History == nil || result.Content.Ancestors == nil {
 				continue
 			}
-			/// Safe guard from deleting items not created by the intended user
-			if result.Content.History.CreatedBy.DisplayName != env.ConfluenceDisplay {
-				continue
-			}
 
 			// Now only delete if this page is in the lineage of the root ancestor
 			for _, ancestor := range result.Content.Ancestors {

hugo2confluence/go.mod

@@ -7,4 +7,8 @@ require (
 	github.com/virtomize/confluence-go-api v1.5.0
 )
 
-require github.com/magefile/mage v1.14.0 // indirect
+require github.com/magefile/mage v1.15.0 // indirect
+
+// This is needed in order to use ther SearchWithNext method
+// See https://github.com/Virtomize/confluence-go-api/pull/69
+replace github.com/virtomize/confluence-go-api => github.com/jdolitsky/confluence-go-api v0.0.0-20250401174407-48bdb8a37784

hugo2confluence/go.sum

@@ -2,14 +2,15 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
+github.com/jdolitsky/confluence-go-api v0.0.0-20250401174407-48bdb8a37784 h1:WfMCvZoFr3myJ/W6ymiJmbNNzfBTwHJn4tjI0P7Vj5c=
+github.com/jdolitsky/confluence-go-api v0.0.0-20250401174407-48bdb8a37784/go.mod h1:a96WPcok5g+7l5LC/ztcrp4cLmrIA1DHxxZSv/iqvsQ=
 github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
+github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sethvargo/go-envconfig v1.1.1 h1:JDu8Q9baIzJf47NPkzhIB6aLYL0vQ+pPypoYrejS9QY=
 github.com/sethvargo/go-envconfig v1.1.1/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/virtomize/confluence-go-api v1.5.0 h1:CRdlL6V/78szvTjZA6XBSevsmqxDkqbDkkrmyoLpa9Y=
-github.com/virtomize/confluence-go-api v1.5.0/go.mod h1:a96WPcok5g+7l5LC/ztcrp4cLmrIA1DHxxZSv/iqvsQ=

hugo2confluence/label/label.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/sethvargo/go-envconfig"
+	confluence "github.com/virtomize/confluence-go-api"
+)
+
+var env = envconfig.MustProcess(context.Background(), &struct {
+	ConfluenceUser     string `env:"CONFLUENCE_USER,required"`
+	ConfluenceToken    string `env:"CONFLUENCE_TOKEN,required"`
+	ConfluenceURL      string `env:"CONFLUENCE_URL,required"`
+	ConfluenceSpace    string `env:"CONFLUENCE_SPACE,required"`
+	ConfluenceAncestor string `env:"CONFLUENCE_ANCESTOR,required"`
+	ConfluenceLabel    string `env:"CONFLUENCE_LABEL,required"`
+}{})
+
+func main() {
+	api, err := confluence.NewAPI(env.ConfluenceURL, env.ConfluenceUser, env.ConfluenceToken)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	next := ""
+	query := confluence.SearchQuery{
+		CQL:    fmt.Sprintf(`space=%q and label in (%q)`, env.ConfluenceSpace, env.ConfluenceLabel),
+		Expand: []string{"content.history,content.ancestors"},
+		Limit:  100,
+	}
+
+	for {
+		resp, err := api.SearchWithNext(query, next)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if resp == nil {
+			break
+		}
+		for _, result := range resp.Results {
+			if result.Content.History == nil || result.Content.Ancestors == nil {
+				continue
+			}
+
+			// Now only delete if this page is in the lineage of the root ancestor
+			for _, ancestor := range result.Content.Ancestors {
+				if ancestor.ID == env.ConfluenceAncestor {
+					id := result.Content.ID
+					// Need to apply the label for cleanup purposes later
+					log.Printf("Applying label \"%s\" to \"%s\" (id:%s)",
+						env.ConfluenceLabel, result.Content.Title, id)
+					labels := []confluence.Label{{Name: env.ConfluenceLabel}}
+					if _, err := api.AddLabels(id, &labels); err != nil {
+						log.Printf("unable to label item %s: %v", id, err)
+					}
+					break
+				}
+			}
+		}
+		next = resp.Links.Next
+		if next == "" {
+			break
+		}
+		log.Printf("Using next page: %s", next)
+	}
+}

hugo2confluence/main.go

@@ -11,7 +11,9 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
+	"time"
 
 	"github.com/sethvargo/go-envconfig"
 	confluence "github.com/virtomize/confluence-go-api"
@@ -24,6 +26,7 @@ var env = envconfig.MustProcess(context.Background(), &struct {
 	ConfluenceSpace    string `env:"CONFLUENCE_SPACE,required"`
 	ConfluenceAncestor string `env:"CONFLUENCE_ANCESTOR,required"`
 	ConfluenceRoot     string `env:"CONFLUENCE_ROOT,required"`
+	ConfluenceLabel    string `env:"CONFLUENCE_LABEL,required"`
 	ConfluenceOrig     string `env:"CONFLUENCE_ORIG"`
 	ConfluenceEdit     string `env:"CONFLUENCE_EDIT"`
 }{})
@@ -47,7 +50,14 @@ func main() {
 
 	log.Printf("Starting site sync to Confluence (url: %s space:%s ancestor:%s)",
 		env.ConfluenceURL, env.ConfluenceSpace, env.ConfluenceAncestor)
-	if err := syncPage(api, rootPage); err != nil {
+	ids, err := syncPage(api, rootPage)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("Cleaning up old Confluence pages (url: %s space:%s ancestor:%s)",
+		env.ConfluenceURL, env.ConfluenceSpace, env.ConfluenceAncestor)
+	if err := cleanupPages(api, ids); err != nil {
 		log.Fatal(err)
 	}
 
@@ -106,13 +116,13 @@ func getPages(rootDir string) ([]page, error) {
 	return pages, nil
 }
 
-func syncPage(api *confluence.API, p page) error {
+func syncPage(api *confluence.API, p page) ([]string, error) {
 	log.Printf("Syncing %s", p.filePath)
 
 	// Generate the content
 	content, err := contentFromFile(p.parentID, p.filePath)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// Check if the page already exists to determine correct version
@@ -135,7 +145,7 @@ func syncPage(api *confluence.API, p page) error {
 			Title:    t,
 		})
 		if err != nil {
-			return err
+			return nil, err
 		}
 		if len(resp.Results) == 0 {
 			break
@@ -172,21 +182,86 @@ func syncPage(api *confluence.API, p page) error {
 	}
 
 	if writeErr != nil {
-		return writeErr
+		return nil, writeErr
 	}
 
 	log.Printf("Wrote page \"%s\" (parent:%s id:%s version:%d url:%s)",
 		written.Title, p.parentID, written.ID, written.Version.Number, written.Links.WebUI)
 
+	// Need to apply the label for cleanup purposes later
+	log.Printf("Applying label \"%s\" to \"%s\" (parent:%s id:%s)",
+		env.ConfluenceLabel, written.Title, p.parentID, written.ID)
+	labels := []confluence.Label{{Name: env.ConfluenceLabel}}
+	if _, err := api.AddLabels(written.ID, &labels); err != nil {
+
+		// Wait 10 seconds and retry (sometimes this endpoint returns a 500?)
+		log.Printf("Got error applying label, retrying in 10 seconds: %s", err)
+		time.Sleep(time.Second * 10)
+		if _, err := api.AddLabels(written.ID, &labels); err != nil {
+			return nil, err
+		}
+	}
+
+	// List of all IDs written in this function call, including recursive ones
+	ids := []string{written.ID}
+
 	// Now recursively write the children
 	// TODO: sync children pages asynchronously?
 	for _, child := range p.children {
 		child.parentID = written.ID
-		if err := syncPage(api, child); err != nil {
-			return err
+		childIds, err := syncPage(api, child)
+		if err != nil {
+			return nil, err
 		}
+		ids = append(childIds, ids...)
 	}
 
+	return ids, nil
+}
+
+func cleanupPages(api *confluence.API, exceptIds []string) error {
+	query := confluence.SearchQuery{
+		CQL:    fmt.Sprintf(`space=%q and label=%q`, env.ConfluenceSpace, env.ConfluenceLabel),
+		Expand: []string{"content.history,content.ancestors"},
+		Limit:  100,
+	}
+	next := ""
+	for {
+		resp, err := api.SearchWithNext(query, next)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if resp == nil {
+			break
+		}
+		for _, result := range resp.Results {
+			if result.Content.History == nil || result.Content.Ancestors == nil {
+				continue
+			}
+
+			// If the ID is one of the ones we have written as part of this run, ignore it
+			id := result.Content.ID
+			if slices.Contains(exceptIds, id) {
+				continue
+			}
+
+			// Now only delete if this page is in the lineage of the root ancestor
+			for _, ancestor := range result.Content.Ancestors {
+				if ancestor.ID == env.ConfluenceAncestor {
+					log.Printf("Cleaning up page %q (id: %s)", result.Content.Title, id)
+					/*if _, err := api.DelContent(id); err != nil {
+						return fmt.Errorf("unable to delete item %s: %v", id, err)
+					}*/
+					break
+				}
+			}
+		}
+		next = resp.Links.Next
+		if next == "" {
+			break
+		}
+		log.Printf("Using next page: %s", next)
+	}
 	return nil
 }