From 6864706394497fd2b54fd83c1874bebccc3a3a78 Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Fri, 16 Aug 2013 15:13:50 +0000 Subject: [PATCH 1/2] Generalise dcs to work on other distributions - Make dcs-unpack and computer-ranking accept a -dist parameter defaulting to 'sid' - Search all components instead of just 'main' --- cmd/compute-ranking/compute-ranking.go | 27 +++---------- cmd/dcs-unpack/unpack.go | 52 ++++++++++++-------------- utils/utils.go | 33 ++++++++++++++++ 3 files changed, 62 insertions(+), 50 deletions(-) create mode 100644 utils/utils.go diff --git a/cmd/compute-ranking/compute-ranking.go b/cmd/compute-ranking/compute-ranking.go index 67ae3d39..3d448590 100644 --- a/cmd/compute-ranking/compute-ranking.go +++ b/cmd/compute-ranking/compute-ranking.go @@ -2,21 +2,21 @@ package main import ( - "compress/bzip2" "database/sql" "flag" "fmt" _ "github.com/lib/pq" - "github.com/mstap/godebiancontrol" + "github.com/Debian/dcs/utils" "log" - "os" - "path/filepath" "strings" ) var mirrorPath = flag.String("mirror_path", "/media/sdd1/debian-source-mirror/", "Path to the debian source mirror (which contains the 'dists' and 'pool' folder)") +var dist = flag.String("dist", + "sid", + "The release to scan") var dryRun = flag.Bool("dry_run", false, "Don’t actually write anything to the database.") var verbose = flag.Bool("verbose", false, "Print ranking information about every package") var popconInstSrc map[string]float32 = make(map[string]float32) @@ -57,21 +57,6 @@ func fillPopconInst() { } -func mustLoadMirroredControlFile(name string) []godebiancontrol.Paragraph { - file, err := os.Open(filepath.Join(*mirrorPath, "dists/sid/main/", name)) - if err != nil { - log.Fatal(err) - } - defer file.Close() - - contents, err := godebiancontrol.Parse(bzip2.NewReader(file)) - if err != nil { - log.Fatal(err) - } - - return contents -} - func main() { flag.Parse() @@ -94,8 +79,8 @@ func main() { } defer update.Close() - sourcePackages := mustLoadMirroredControlFile("source/Sources.bz2") - binaryPackages := mustLoadMirroredControlFile("binary-amd64/Packages.bz2") + sourcePackages := utils.MustLoadMirroredControlFile(*mirrorPath, *dist, "source/Sources.bz2") + binaryPackages := utils.MustLoadMirroredControlFile(*mirrorPath, *dist, "binary-amd64/Packages.bz2") reverseDeps := make(map[string]uint) for _, pkg := range binaryPackages { diff --git a/cmd/dcs-unpack/unpack.go b/cmd/dcs-unpack/unpack.go index 1c80d949..2a3fc90a 100644 --- a/cmd/dcs-unpack/unpack.go +++ b/cmd/dcs-unpack/unpack.go @@ -1,10 +1,9 @@ package main import ( - "compress/bzip2" "flag" "fmt" - "github.com/mstap/godebiancontrol" + "github.com/Debian/dcs/utils" "log" "os" "os/exec" @@ -22,6 +21,9 @@ var oldUnpackPath = flag.String("old_unpacked_path", var newUnpackPath = flag.String("new_unpacked_path", "/dcs-ssd/unpacked-new/", "Path to the unpacked debian source mirror") +var dist = flag.String("dist", + "sid", + "The release to scan") // Copies directories by hard-linking all files inside, // necessary since hard-links on directories are not possible. @@ -48,17 +50,7 @@ func linkDirectory(oldPath, newPath string) error { func main() { flag.Parse() - // Walk through all source packages - file, err := os.Open(path.Join(*mirrorPath, "/dists/sid/main/source/Sources.bz2")) - if err != nil { - log.Fatal(err) - } - defer file.Close() - - sourcePackages, err := godebiancontrol.Parse(bzip2.NewReader(file)) - if err != nil { - log.Fatal(err) - } + sourcePackages := utils.MustLoadMirroredControlFile(*mirrorPath, *dist, "source/Sources.bz2") if err := os.Mkdir(*newUnpackPath, 0775); err != nil && !os.IsExist(err) { log.Fatal(err) @@ -75,30 +67,30 @@ func main() { oldPath := path.Join(*oldUnpackPath, dir) newPath := path.Join(*newUnpackPath, dir) - // Check whether the directory exists in the old "unpacked" directory - if _, err := os.Stat(oldPath); err == nil { + // Check whether the directory exists in the old "unpacked" + // directory and hardlink only if the new path doesn't exist + // (to avoid wasted time hardlinking in case of partial runs) + _, oldErr := os.Stat(oldPath) + _, newErr := os.Stat (newPath) + if oldErr == nil && newErr != nil { log.Printf("hardlink %s\n", dir) // If so, just hardlink it to save space and computing time. if err := linkDirectory(oldPath, newPath); err != nil { log.Fatal(err) } - } else { + } else if oldErr != nil && newErr != nil { log.Printf("unpack %s\n", dir) - files := strings.Split(pkg["Files"], "\n") - filepath := "" - for _, line := range files { - if !strings.HasSuffix(line, ".dsc") { - continue - } - + files := strings.Split(pkg["Files"], "\n") + filepath := "" + for _, line := range files { + if !strings.HasSuffix(line, ".dsc") { + continue + } + parts := strings.Split(line, " ") file := parts[len(parts)-1] - prefix := string(file[0]) - if strings.HasPrefix(file, "lib") { - prefix = "lib" + string(file[3]) - } - filepath = path.Join(*mirrorPath, "pool", "main", prefix, pkg["Package"], file) - } + filepath = path.Join(*mirrorPath, pkg["Directory"], file) + } if filepath == "" { log.Fatalf("Package %s contains no dsc file, cannot unpack\n", pkg["Package"]) @@ -114,6 +106,8 @@ func main() { if err := cmd.Run(); err != nil { log.Fatal(err) } + } else { + log.Printf("Skip unpack of %s\n", dir) } } } diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 00000000..f90f2713 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,33 @@ +package utils + +import ( + "compress/bzip2" + "github.com/mstap/godebiancontrol" + "log" + "os" + "path/filepath" +) + +func MustLoadMirroredControlFile(mirrorPath string, dist string, name string) []godebiancontrol.Paragraph { + var base = filepath.Join(mirrorPath, "dists", dist) + files, err := os.Open(base) + if err != nil { + log.Fatal(err) + } + fi, err := files.Readdir(-1) + var contents = make([]godebiancontrol.Paragraph, 0) + for _, file := range fi { + if !file.IsDir() { + continue + } + file, err := os.Open(filepath.Join(base, file.Name(), name)) + contents_new, err := godebiancontrol.Parse(bzip2.NewReader(file)) + if err != nil { + log.Fatal(err) + } + contents = append(contents, contents_new...) + defer file.Close() + } + + return contents +} From 47bed5db3dc82d8e3745d3e3f14c4b6601cddbfe Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Sat, 17 Aug 2013 11:17:17 +0000 Subject: [PATCH 2/2] Whitespace fix --- cmd/dcs-unpack/unpack.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/dcs-unpack/unpack.go b/cmd/dcs-unpack/unpack.go index 2a3fc90a..d1d11104 100644 --- a/cmd/dcs-unpack/unpack.go +++ b/cmd/dcs-unpack/unpack.go @@ -80,17 +80,17 @@ func main() { } } else if oldErr != nil && newErr != nil { log.Printf("unpack %s\n", dir) - files := strings.Split(pkg["Files"], "\n") - filepath := "" - for _, line := range files { - if !strings.HasSuffix(line, ".dsc") { - continue - } + files := strings.Split(pkg["Files"], "\n") + filepath := "" + for _, line := range files { + if !strings.HasSuffix(line, ".dsc") { + continue + } - parts := strings.Split(line, " ") - file := parts[len(parts)-1] - filepath = path.Join(*mirrorPath, pkg["Directory"], file) - } + parts := strings.Split(line, " ") + file := parts[len(parts)-1] + filepath = path.Join(*mirrorPath, pkg["Directory"], file) + } if filepath == "" { log.Fatalf("Package %s contains no dsc file, cannot unpack\n", pkg["Package"])