Add project cache

This adds cache for project packages. It can be enabled with
"project_cache: 1" in newtrc.yml.

Once enabled, packages list for each repo will be stored in .cache
directory and read from there on subsequent executions. This allows to
substantially strip time required to find packages in each repo. Cache
will be used as long as current repo hash matches the cache.

Author: andrzej-kaczmarek

newt/cache/cache.go

@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package cache
+
+import (
+	"encoding/gob"
+	"fmt"
+	"mynewt.apache.org/newt/newt/interfaces"
+	"mynewt.apache.org/newt/newt/repo"
+	"mynewt.apache.org/newt/util"
+	"os"
+	"path/filepath"
+)
+
+type ProjectCache struct {
+	baseDir string
+}
+
+func InitCache(projDir string) *ProjectCache {
+	pc := ProjectCache{}
+	pc.baseDir = filepath.Join(projDir, ".cache")
+
+	if _, err := os.Stat(pc.baseDir); os.IsNotExist(err) {
+		if err := os.Mkdir(pc.baseDir, 0754); err != nil {
+			return nil
+		}
+	}
+
+	return &pc
+}
+
+func (pc *ProjectCache) getPackagesFile(repo *repo.Repo) string {
+	return fmt.Sprintf(filepath.Join(pc.baseDir, repo.Name()))
+}
+
+func (pc *ProjectCache) AddPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface) {
+	cacheName := pc.getPackagesFile(repo)
+	var dirList []string
+
+	hash, err := repo.CurrentHash()
+	if err != nil {
+		return
+	}
+
+	for _, v := range pkgMap {
+		dirList = append(dirList, v.BasePath())
+	}
+
+	f, err := os.Create(cacheName)
+	if err != nil {
+		util.OneTimeWarning("Failed to create cache file for \"%s\"", repo.Name())
+		return
+	}
+
+	defer f.Close()
+
+	enc := gob.NewEncoder(f)
+	enc.Encode(hash)
+	enc.Encode(dirList)
+}
+
+func (pc *ProjectCache) GetPackagesDirs(repo *repo.Repo) []string {
+	cacheName := pc.getPackagesFile(repo)
+	var dirList []string
+
+	f, err := os.Open(cacheName)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			util.OneTimeWarning("Failed to open cache file for \"%s\"", repo.Name())
+		}
+		return nil
+	}
+
+	defer f.Close()
+
+	var hash string
+
+	enc := gob.NewDecoder(f)
+	err = enc.Decode(&hash)
+	if err != nil {
+		util.OneTimeWarning("Failed to read cache for \"%s\"", repo.Name())
+		return nil
+	}
+
+	currHash, _ := repo.CurrentHash()
+	if hash != currHash {
+		return nil
+	}
+
+	err = enc.Decode(&dirList)
+	if err != nil {
+		util.OneTimeWarning("Failed to read cache for \"%s\"", repo.Name())
+		return nil
+	}
+
+	return dirList
+}

newt/pkg/localpackage.go

@@ -449,8 +449,34 @@ func LocalPackageSpecialName(dirName string) bool {
 	return ok
 }
 
+func ReadPackage(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface,
+	pkgPath string) ([]string, error) {
+
+	var warnings []string
+
+	pkg, err := LoadLocalPackage(repo, pkgPath)
+	if err != nil {
+		warnings = append(warnings, err.Error())
+		return warnings, nil
+	}
+
+	if oldPkg, ok := pkgMap[pkg.Name()]; ok {
+		oldlPkg := oldPkg.(*LocalPackage)
+		warnings = append(warnings,
+			fmt.Sprintf("Multiple packages with same pkg.name=%s "+
+				"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
+				oldlPkg.BasePath(), pkg.BasePath()))
+
+		return warnings, nil
+	}
+
+	pkgMap[pkg.Name()] = pkg
+
+	return warnings, nil
+}
+
 func ReadLocalPackageRecursive(repo *repo.Repo,
-	pkgList map[string]interfaces.PackageInterface, basePath string,
+	pkgMap map[string]interfaces.PackageInterface, basePath string,
 	pkgName string, searchedMap map[string]struct{}) ([]string, error) {
 
 	var warnings []string
@@ -465,7 +491,7 @@ func ReadLocalPackageRecursive(repo *repo.Repo,
 			continue
 		}
 
-		subWarnings, err := ReadLocalPackageRecursive(repo, pkgList,
+		subWarnings, err := ReadLocalPackageRecursive(repo, pkgMap,
 			basePath, filepath.Join(pkgName, name), searchedMap)
 		warnings = append(warnings, subWarnings...)
 		if err != nil {
@@ -477,39 +503,21 @@ func ReadLocalPackageRecursive(repo *repo.Repo,
 		return warnings, nil
 	}
 
-	pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName))
-	if err != nil {
-		warnings = append(warnings, err.Error())
-		return warnings, nil
-	}
+	var subWarnings []string
+	subWarnings, err = ReadPackage(repo, pkgMap, filepath.Join(basePath, pkgName))
+	warnings = append(warnings, subWarnings...)
 
-	if oldPkg, ok := pkgList[pkg.Name()]; ok {
-		oldlPkg := oldPkg.(*LocalPackage)
-		warnings = append(warnings,
-			fmt.Sprintf("Multiple packages with same pkg.name=%s "+
-				"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
-				oldlPkg.BasePath(), pkg.BasePath()))
-
-		return warnings, nil
-	}
-
-	pkgList[pkg.Name()] = pkg
-
-	return warnings, nil
+	return warnings, err
 }
 
-func ReadLocalPackages(repo *repo.Repo, basePath string) (
-	*map[string]interfaces.PackageInterface, []string, error) {
-
-	pkgMap := &map[string]interfaces.PackageInterface{}
-
+func ReadLocalPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface, basePath string) ([]string, error) {
 	// Keep track of which directories we have traversed.  Prevent infinite
 	// loops caused by symlink cycles by not inspecting the same directory
 	// twice.
 	searchedMap := map[string]struct{}{}
 
-	warnings, err := ReadLocalPackageRecursive(repo, *pkgMap,
+	warnings, err := ReadLocalPackageRecursive(repo, pkgMap,
 		basePath, "", searchedMap)
 
-	return pkgMap, warnings, err
+	return warnings, err
 }

newt/project/project.go

@@ -21,6 +21,7 @@ package project
 
 import (
 	"fmt"
+	"mynewt.apache.org/newt/newt/cache"
 	"os"
 	"path"
 	"path/filepath"
@@ -57,6 +58,8 @@ type Project struct {
 	// Base path of the project
 	BasePath string
 
+	cache *cache.ProjectCache
+
 	packages interfaces.PackageList
 
 	// Contains all the repos that form this project.  Each repo is in one of
@@ -634,6 +637,13 @@ func (proj *Project) Init(dir string, download bool) error {
 		return err
 	}
 
+	if util.EnableProjectCache {
+		proj.cache = cache.InitCache(proj.BasePath)
+		if proj.cache == nil {
+			util.OneTimeWarning("Failed to initialize project cache")
+		}
+	}
+
 	return nil
 }
 
@@ -741,12 +751,37 @@ func (proj *Project) loadPackageList() error {
 	// packages / store them in the project package list.
 	repos := proj.Repos()
 	for name, repo := range repos {
-		list, warnings, err := pkg.ReadLocalPackages(repo, repo.Path())
-		if err == nil {
-			proj.packages[name] = list
+		pkgMap := &map[string]interfaces.PackageInterface{}
+
+		var dirList []string
+
+		if proj.cache != nil {
+			dirList = proj.cache.GetPackagesDirs(repo)
 		}
 
-		proj.warnings = append(proj.warnings, warnings...)
+		if proj.cache != nil && dirList != nil {
+			log.Debug("Using cache for packages in \"%s\"\n", repo.Name())
+			for _, pkgPath := range dirList {
+				warnings, err := pkg.ReadPackage(repo, *pkgMap, pkgPath)
+				if err == nil {
+					proj.packages[name] = pkgMap
+				}
+
+				proj.warnings = append(proj.warnings, warnings...)
+			}
+		} else {
+			log.Debug("Not using cache for packages in \"%s\"\n", repo.Name())
+			warnings, err := pkg.ReadLocalPackages(repo, *pkgMap, repo.Path())
+			if err == nil {
+				proj.packages[name] = pkgMap
+			}
+
+			if proj.cache != nil {
+				proj.cache.AddPackages(repo, *pkgMap)
+			}
+
+			proj.warnings = append(proj.warnings, warnings...)
+		}
 	}
 
 	return nil

newt/settings/settings.go

@@ -59,6 +59,7 @@ func processNewtrc(yc ycfg.YCfg) {
 
 	util.SkipNewtCompat, _ = yc.GetValBoolDflt("skip_newt_compat", nil, false)
 	util.SkipSyscfgRepoHash, _ = yc.GetValBoolDflt("skip_syscfg_repo_hash", nil, false)
+	util.EnableProjectCache, _ = yc.GetValBoolDflt("project_cache", nil, false)
 }
 
 func readNewtrc() ycfg.YCfg {

util/util.go

@@ -51,6 +51,7 @@ var ShallowCloneDepth int
 var logFile *os.File
 var SkipNewtCompat bool
 var SkipSyscfgRepoHash bool
+var EnableProjectCache bool
 
 func ParseEqualsPair(v string) (string, string, error) {
 	s := strings.Split(v, "=")