Fetch dev tree from GitHub

.gitignore

@@ -11,4 +11,6 @@ logback.xml
 *.DS_Store
 web/src/main/javascript/public/
 web/src/main/resources/static 
-web/src/main/go/docs
+web/src/main/go/docs
+trees/oncotree_development.json.etag
+web/src/main/javascript/public/assets

Dockerfile

@@ -31,4 +31,5 @@ ENV STATIC_DIR=/root/backend/frontend/static
 
 EXPOSE 8080
 ENV GIN_MODE=release
+ENV APP_ENV=production
 CMD ["./main"]

web/src/main/go/cmd/server/main.go

@@ -298,12 +298,13 @@ func tumorTypesTreeHandler(c *gin.Context) {
 
 	treeFile := fmt.Sprintf("%s.json", resolvedVersion)
 
-	raw, err := os.ReadFile(filepath.Join(TreeDir, treeFile))
+	raw, err := internal.ReadTreeRaw(treeFile)
 	if err != nil {
 		c.Error(fmt.Errorf("Failed to read raw tree file '%s': %w", treeFile, err))
 		c.String(http.StatusServiceUnavailable, "Required data source unavailable")
 		return
 	}
+
 	c.Data(http.StatusOK, "application/json", raw)
 }
 

web/src/main/go/internal/constants.go

@@ -19,6 +19,10 @@ func init() {
 	TSV_FILES_PATH = filepath.Join(TREE_FILES_PATH, "tsv")
 }
 
+const (
+	DEV_TREE_GITHUB_RAW_URL = "https://raw.githubusercontent.com/cBioPortal/oncotree/master/trees/oncotree_development.json"
+)
+
 const (
 	LEGACY_TREE_IDENTIFIER        = "oncotree_legacy_1.1"
 	CANDIDATE_TREE_IDENTIFIER     = "oncotree_candidate_release"

web/src/main/go/internal/utils.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -17,7 +18,7 @@ import (
 )
 
 func ReadTreeFromFile(name string) (Tree, error) {
-	treeBytes, err := os.ReadFile(GetTreeFilepath(name))
+	treeBytes, err := ReadTreeRaw(name)
 	if err != nil {
 		return nil, fmt.Errorf("error reading file '%v': %v", name, err)
 	}
@@ -202,3 +203,72 @@ func (file *DatedFile) GetDatedFilenameWithoutExtension() string {
 	name := strings.Replace(file.Name, ".txt", "", 1)
 	return strings.Replace(name, ".json", "", 1)
 }
+
+func fetchDevTreeIfChanged(devTreePath string) error {
+	cacheDir := filepath.Join(TREE_FILES_PATH, "..", "cache")
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return fmt.Errorf("failed to create cache dir: %w", err)
+	}
+
+	base := filepath.Base(devTreePath)
+	tmpPath := filepath.Join(cacheDir, base+".tmp")
+	etagPath := filepath.Join(cacheDir, base+".etag")
+
+	req, err := http.NewRequest(http.MethodGet, DEV_TREE_GITHUB_RAW_URL, nil)
+	if err != nil {
+		return err
+	}
+
+	if etag, err := os.ReadFile(etagPath); err == nil {
+		req.Header.Set("If-None-Match", string(etag))
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusNotModified:
+		return nil
+
+	case http.StatusOK:
+		out, err := os.Create(tmpPath)
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+
+		if _, err := io.Copy(out, resp.Body); err != nil {
+			return err
+		}
+
+		if err := os.Rename(tmpPath, devTreePath); err != nil {
+			return err
+		}
+
+		if etag := resp.Header.Get("ETag"); etag != "" {
+			if err := os.WriteFile(etagPath, []byte(etag), 0644); err != nil {
+				return fmt.Errorf("failed to write etag: %w", err)
+			}
+		}
+
+		return nil
+
+	default:
+		return fmt.Errorf("unexpected status from GitHub: %s", resp.Status)
+	}
+}
+
+func ReadTreeRaw(name string) ([]byte, error) {
+	appEnv := os.Getenv("APP_ENV")
+	if name == DEV_TREE_IDENTIFIER+".json" && appEnv == "production" {
+		devTreePath := filepath.Join(TREE_FILES_PATH, name)
+		if err := fetchDevTreeIfChanged(devTreePath); err != nil {
+			return nil, err
+		}
+	}
+
+	return os.ReadFile(GetTreeFilepath(name))
+}