From f9c2c3201fbf11d4ce6db087706393234e7af810 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Thu, 27 Oct 2022 09:51:10 -0500 Subject: [PATCH 1/2] sqlite: add fs.FS support This makes use of functionality added to the sqlite library to use fs.FS implementations. Signed-off-by: Hank Donnay Change-Id: Icf0f7832e08908415f8d1da132e5e3246a6a6964 JJ: See-Also: CLAIRDEV-NNNN JJ: Closes: #NNNN --- internal/rpm/sqlite/sqlite.go | 62 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/internal/rpm/sqlite/sqlite.go b/internal/rpm/sqlite/sqlite.go index 73cf38a13..5c1bce77a 100644 --- a/internal/rpm/sqlite/sqlite.go +++ b/internal/rpm/sqlite/sqlite.go @@ -9,16 +9,19 @@ import ( "errors" "fmt" "io" + "io/fs" "iter" "net/url" "runtime" _ "modernc.org/sqlite" // register the sqlite driver + "modernc.org/sqlite/vfs" ) // RPMDB is a handle to a SQLite RPM database. type RPMDB struct { - db *sql.DB + db *sql.DB + vfs *vfs.FS } // Open opens the named SQLite database and interprets it as an RPM @@ -56,13 +59,68 @@ func Open(f string) (*RPMDB, error) { return &rdb, nil } +// OpenFS opens the named SQLite database in the context of "sys" and interprets +// it as an RPM database. +// +// The returned RPMDB struct must have its Close method called, or the process +// may panic. +func OpenFS(sys fs.FS, f string) (*RPMDB, error) { + name, vfs, err := vfs.New(sys) + if err != nil { + return nil, fmt.Errorf("sqlite: vfs creation failed: %w", err) + } + ok := false + defer func() { + if !ok { + vfs.Close() + } + }() + u := url.URL{ + Scheme: `file`, + Opaque: f, + RawQuery: url.Values{ + "vfs": {name}, + "immutable": {"1"}, + "_pragma": { + "foreign_keys(1)", + "query_only(1)", + }, + }.Encode(), + } + db, err := sql.Open(`sqlite`, u.String()) + if err != nil { + return nil, fmt.Errorf("sqlite: unable to open db: %w", err) + } + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("sqlite: unable to ping db: %w", err) + } + rdb := RPMDB{ + db: db, + vfs: vfs, + } + ok = true + _, file, line, _ := runtime.Caller(1) + runtime.SetFinalizer(&rdb, func(rdb *RPMDB) { + panic(fmt.Sprintf("%s:%d: RPM db not closed", file, line)) + }) + return &rdb, nil +} + // Close releases held resources. // // This must be called when the RPMDB is no longer needed, or the // process may panic. func (db *RPMDB) Close() error { runtime.SetFinalizer(db, nil) - return db.db.Close() + return errors.Join( + db.db.Close(), + func() error { + if db.vfs != nil { + return db.vfs.Close() + } + return nil + }(), + ) } // Headers returns an iterator over all RPM headers in the database. From bbe51e439c533df516871bdfa58be57fdcf4a237 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Thu, 27 Oct 2022 09:53:08 -0500 Subject: [PATCH 2/2] rpm: avoid spooling sqlite database Signed-off-by: Hank Donnay Change-Id: Iad2ca9ac10c24be386b98034da1ba1aa6a6a6964 JJ: See-Also: CLAIRDEV-NNNN JJ: Closes: #NNNN --- internal/rpm/find.go | 109 +++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/internal/rpm/find.go b/internal/rpm/find.go index b31d4cbc4..802a630b4 100644 --- a/internal/rpm/find.go +++ b/internal/rpm/find.go @@ -1,6 +1,7 @@ package rpm import ( + "bytes" "context" "errors" "fmt" @@ -160,89 +161,45 @@ func OpenDB(ctx context.Context, sys fs.FS, found FoundDB) (*Database, error) { // TODO(hank) Cook up some test against passing the wrong [fs.FS]. Don't use // the unique package. - f, err := sys.Open(found.filename()) - if err != nil { - return nil, fmt.Errorf("internal/rpm: unable to open %s db: %w", found.kind.String(), err) - } - defer func() { - if err := f.Close(); err != nil { - slog.WarnContext(ctx, "unable to close db", "kind", found, "reason", err) - } - }() - cleanup := &databaseCleanup{} db := Database{ pkgdb: found.String(), cleanup: cleanup, } - spool, err := os.CreateTemp(os.TempDir(), fmt.Sprintf(`rpm.package.%v.`, found.kind)) - if err != nil { - return nil, fmt.Errorf("internal/rpm: error spooling db: %w", err) - } - log := slog.With("file", spool.Name()) - cleanup.spool = spool - log.DebugContext(ctx, "copying db out of fs.FS") - - // Need to have the file linked into the filesystem for the sqlite package. - // - // See [this post] for an idea on working around it: - // - // int sqlite_fdopen( - // int fd, - // sqlite3 **connection) - // { - // char uri[48]; - // - // snprintf(uri, sizeof uri, "file:///dev/fd/%d?immutable=1", fd); - // return sqlite3_open_v2( - // uri, - // connection, - // SQLITE_OPEN_READONLY | SQLITE_OPEN_URI, - // NULL); - // } - // - // [this post]: https://sqlite.org/forum/info/57aaaf20cf703d301fed5aeaef59e70723f1d9454fb3a4e6383b2bfac6897e5a - if _, err := io.Copy(spool, f); err != nil { - if err := spool.Close(); err != nil { - log.WarnContext(ctx, "unable to close spool", "reason", err) - } - return nil, fmt.Errorf("internal/rpm: error spooling db: %w", err) - } - if err := spool.Sync(); err != nil { - log.WarnContext(ctx, "unable to sync spool; results may be Weird", "reason", err) - } - switch found.kind { case kindSQLite: - sdb, err := sqlite.Open(spool.Name()) + sdb, err := sqlite.OpenFS(sys, found.filename()) if err != nil { - if err := spool.Close(); err != nil { - log.WarnContext(ctx, "unable to close spool", "reason", err) - } - if err := os.Remove(spool.Name()); err != nil { - log.WarnContext(ctx, "unable to remove spool", "reason", err) - } return nil, fmt.Errorf("internal/rpm: unable to open sqlite db: %w", err) } db.headers = sdb cleanup.close = sdb.Close - case kindBDB: - var bpdb bdb.PackageDB - if err := bpdb.Parse(spool); err != nil { - return nil, fmt.Errorf("internal/rpm: error parsing bdb db: %w", err) + case kindBDB, kindNDB: + r, err := db.openOrBuffer(ctx, sys, found) + if err != nil { + return nil, fmt.Errorf("internal/rpm: unable to open %s db: %w", found.kind.String(), err) } - db.headers = &bpdb - case kindNDB: - var npdb ndb.PackageDB - if err := npdb.Parse(spool); err != nil { - return nil, fmt.Errorf("internal/rpm: error parsing ndb db: %w", err) + switch found.kind { + case kindBDB: + var bpdb bdb.PackageDB + if err := bpdb.Parse(r); err != nil { + return nil, fmt.Errorf("internal/rpm: error parsing bdb db: %w", err) + } + db.headers = &bpdb + case kindNDB: + var npdb ndb.PackageDB + if err := npdb.Parse(r); err != nil { + return nil, fmt.Errorf("internal/rpm: error parsing ndb db: %w", err) + } + db.headers = &npdb + default: + panic("unreachable") } - db.headers = &npdb default: panic("unreachable") } - log.DebugContext(ctx, "opened database", "db", found) + slog.DebugContext(ctx, "opened database", "db", found) if v, ok := db.headers.(validator); ok { if err := v.Validate(ctx); err != nil { @@ -253,6 +210,28 @@ func OpenDB(ctx context.Context, sys fs.FS, found FoundDB) (*Database, error) { return &db, nil } +func (db *Database) openOrBuffer(_ context.Context, sys fs.FS, found FoundDB) (io.ReaderAt, error) { + f, err := sys.Open(found.filename()) + if err != nil { + return nil, err + } + r, ok := f.(io.ReaderAt) + if ok { + db.cleanup = f + return r, nil + } + defer f.Close() + + var buf bytes.Buffer + if fi, err := f.Stat(); err == nil { + buf.Grow(int(fi.Size())) + } + if _, err := io.Copy(&buf, f); err != nil { + return nil, err + } + return bytes.NewReader(buf.Bytes()), nil +} + type databaseCleanup struct { spool *os.File close func() error