From 6c47e38bb5e9945542196bde597d742db4afef0b Mon Sep 17 00:00:00 2001 From: Andrzej Kaczmarek Date: Thu, 31 Mar 2022 12:44:33 +0200 Subject: [PATCH] Add project cache This adds cache for project packages. It can be enabled with "project_cache: 1" in newtrc.yml. Once enabled, packages list for each repo will be stored in .cache directory and read from there on subsequent executions. This allows to substantially strip time required to find packages in each repo. Cache will be used as long as current repo hash matches the cache. --- newt/cache/cache.go | 114 ++++++++++++++++++++++++++++++++++++++ newt/pkg/localpackage.go | 62 ++++++++++++--------- newt/project/project.go | 43 ++++++++++++-- newt/settings/settings.go | 1 + util/util.go | 1 + 5 files changed, 190 insertions(+), 31 deletions(-) create mode 100644 newt/cache/cache.go diff --git a/newt/cache/cache.go b/newt/cache/cache.go new file mode 100644 index 0000000000..b5d7517dbc --- /dev/null +++ b/newt/cache/cache.go @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package cache + +import ( + "encoding/gob" + "fmt" + "mynewt.apache.org/newt/newt/interfaces" + "mynewt.apache.org/newt/newt/repo" + "mynewt.apache.org/newt/util" + "os" + "path/filepath" +) + +type ProjectCache struct { + baseDir string +} + +func InitCache(projDir string) *ProjectCache { + pc := ProjectCache{} + pc.baseDir = filepath.Join(projDir, ".cache") + + if _, err := os.Stat(pc.baseDir); os.IsNotExist(err) { + if err := os.Mkdir(pc.baseDir, 0754); err != nil { + return nil + } + } + + return &pc +} + +func (pc *ProjectCache) getPackagesFile(repo *repo.Repo) string { + return fmt.Sprintf(filepath.Join(pc.baseDir, repo.Name())) +} + +func (pc *ProjectCache) AddPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface) { + cacheName := pc.getPackagesFile(repo) + var dirList []string + + hash, err := repo.CurrentHash() + if err != nil { + return + } + + for _, v := range pkgMap { + dirList = append(dirList, v.BasePath()) + } + + f, err := os.Create(cacheName) + if err != nil { + util.OneTimeWarning("Failed to create cache file for \"%s\"", repo.Name()) + return + } + + defer f.Close() + + enc := gob.NewEncoder(f) + enc.Encode(hash) + enc.Encode(dirList) +} + +func (pc *ProjectCache) GetPackagesDirs(repo *repo.Repo) []string { + cacheName := pc.getPackagesFile(repo) + var dirList []string + + f, err := os.Open(cacheName) + if err != nil { + if !os.IsNotExist(err) { + util.OneTimeWarning("Failed to open cache file for \"%s\"", repo.Name()) + } + return nil + } + + defer f.Close() + + var hash string + + enc := gob.NewDecoder(f) + err = enc.Decode(&hash) + if err != nil { + util.OneTimeWarning("Failed to read cache for \"%s\"", repo.Name()) + return nil + } + + currHash, _ := repo.CurrentHash() + if hash != currHash { + return nil + } + + err = enc.Decode(&dirList) + if err != nil { + util.OneTimeWarning("Failed to read cache for \"%s\"", repo.Name()) + return nil + } + + return dirList +} diff --git a/newt/pkg/localpackage.go b/newt/pkg/localpackage.go index a68cbf2c7d..f566b064f7 100644 --- a/newt/pkg/localpackage.go +++ b/newt/pkg/localpackage.go @@ -449,8 +449,34 @@ func LocalPackageSpecialName(dirName string) bool { return ok } +func ReadPackage(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface, + pkgPath string) ([]string, error) { + + var warnings []string + + pkg, err := LoadLocalPackage(repo, pkgPath) + if err != nil { + warnings = append(warnings, err.Error()) + return warnings, nil + } + + if oldPkg, ok := pkgMap[pkg.Name()]; ok { + oldlPkg := oldPkg.(*LocalPackage) + warnings = append(warnings, + fmt.Sprintf("Multiple packages with same pkg.name=%s "+ + "in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(), + oldlPkg.BasePath(), pkg.BasePath())) + + return warnings, nil + } + + pkgMap[pkg.Name()] = pkg + + return warnings, nil +} + func ReadLocalPackageRecursive(repo *repo.Repo, - pkgList map[string]interfaces.PackageInterface, basePath string, + pkgMap map[string]interfaces.PackageInterface, basePath string, pkgName string, searchedMap map[string]struct{}) ([]string, error) { var warnings []string @@ -465,7 +491,7 @@ func ReadLocalPackageRecursive(repo *repo.Repo, continue } - subWarnings, err := ReadLocalPackageRecursive(repo, pkgList, + subWarnings, err := ReadLocalPackageRecursive(repo, pkgMap, basePath, filepath.Join(pkgName, name), searchedMap) warnings = append(warnings, subWarnings...) if err != nil { @@ -477,39 +503,21 @@ func ReadLocalPackageRecursive(repo *repo.Repo, return warnings, nil } - pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName)) - if err != nil { - warnings = append(warnings, err.Error()) - return warnings, nil - } + var subWarnings []string + subWarnings, err = ReadPackage(repo, pkgMap, filepath.Join(basePath, pkgName)) + warnings = append(warnings, subWarnings...) - if oldPkg, ok := pkgList[pkg.Name()]; ok { - oldlPkg := oldPkg.(*LocalPackage) - warnings = append(warnings, - fmt.Sprintf("Multiple packages with same pkg.name=%s "+ - "in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(), - oldlPkg.BasePath(), pkg.BasePath())) - - return warnings, nil - } - - pkgList[pkg.Name()] = pkg - - return warnings, nil + return warnings, err } -func ReadLocalPackages(repo *repo.Repo, basePath string) ( - *map[string]interfaces.PackageInterface, []string, error) { - - pkgMap := &map[string]interfaces.PackageInterface{} - +func ReadLocalPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface, basePath string) ([]string, error) { // Keep track of which directories we have traversed. Prevent infinite // loops caused by symlink cycles by not inspecting the same directory // twice. searchedMap := map[string]struct{}{} - warnings, err := ReadLocalPackageRecursive(repo, *pkgMap, + warnings, err := ReadLocalPackageRecursive(repo, pkgMap, basePath, "", searchedMap) - return pkgMap, warnings, err + return warnings, err } diff --git a/newt/project/project.go b/newt/project/project.go index 879e3922fd..93ad16d89f 100644 --- a/newt/project/project.go +++ b/newt/project/project.go @@ -21,6 +21,7 @@ package project import ( "fmt" + "mynewt.apache.org/newt/newt/cache" "os" "path" "path/filepath" @@ -57,6 +58,8 @@ type Project struct { // Base path of the project BasePath string + cache *cache.ProjectCache + packages interfaces.PackageList // Contains all the repos that form this project. Each repo is in one of @@ -634,6 +637,13 @@ func (proj *Project) Init(dir string, download bool) error { return err } + if util.EnableProjectCache { + proj.cache = cache.InitCache(proj.BasePath) + if proj.cache == nil { + util.OneTimeWarning("Failed to initialize project cache") + } + } + return nil } @@ -741,12 +751,37 @@ func (proj *Project) loadPackageList() error { // packages / store them in the project package list. repos := proj.Repos() for name, repo := range repos { - list, warnings, err := pkg.ReadLocalPackages(repo, repo.Path()) - if err == nil { - proj.packages[name] = list + pkgMap := &map[string]interfaces.PackageInterface{} + + var dirList []string + + if proj.cache != nil { + dirList = proj.cache.GetPackagesDirs(repo) } - proj.warnings = append(proj.warnings, warnings...) + if proj.cache != nil && dirList != nil { + log.Debug("Using cache for packages in \"%s\"\n", repo.Name()) + for _, pkgPath := range dirList { + warnings, err := pkg.ReadPackage(repo, *pkgMap, pkgPath) + if err == nil { + proj.packages[name] = pkgMap + } + + proj.warnings = append(proj.warnings, warnings...) + } + } else { + log.Debug("Not using cache for packages in \"%s\"\n", repo.Name()) + warnings, err := pkg.ReadLocalPackages(repo, *pkgMap, repo.Path()) + if err == nil { + proj.packages[name] = pkgMap + } + + if proj.cache != nil { + proj.cache.AddPackages(repo, *pkgMap) + } + + proj.warnings = append(proj.warnings, warnings...) + } } return nil diff --git a/newt/settings/settings.go b/newt/settings/settings.go index c70b58bb5b..7137ed4118 100644 --- a/newt/settings/settings.go +++ b/newt/settings/settings.go @@ -59,6 +59,7 @@ func processNewtrc(yc ycfg.YCfg) { util.SkipNewtCompat, _ = yc.GetValBoolDflt("skip_newt_compat", nil, false) util.SkipSyscfgRepoHash, _ = yc.GetValBoolDflt("skip_syscfg_repo_hash", nil, false) + util.EnableProjectCache, _ = yc.GetValBoolDflt("project_cache", nil, false) } func readNewtrc() ycfg.YCfg { diff --git a/util/util.go b/util/util.go index 41632bccf0..b247915c51 100644 --- a/util/util.go +++ b/util/util.go @@ -51,6 +51,7 @@ var ShallowCloneDepth int var logFile *os.File var SkipNewtCompat bool var SkipSyscfgRepoHash bool +var EnableProjectCache bool func ParseEqualsPair(v string) (string, string, error) { s := strings.Split(v, "=")