Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

Commit 56c5e91

Browse files
erizocosmicoerizocosmico
committed
plumbing: packfile, open and close packfile on FSObject reads
Signed-off-by: Miguel Molina <miguel@erizocosmi.co>
1 parent d93b386 commit 56c5e91

File tree

9 files changed

+174
-75
lines changed

9 files changed

+174
-75
lines changed

‎plumbing/format/packfile/encoder_advanced_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"math/rand"
77
"testing"
88

9-
"gopkg.in/src-d/go-billy.v3/memfs"
9+
"gopkg.in/src-d/go-billy.v4/memfs"
1010
"gopkg.in/src-d/go-git.v4/plumbing"
1111
"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
1212
. "gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
@@ -84,7 +84,8 @@ func (s *EncoderAdvancedSuite) testEncodeDecode(
8484
encodeHash, err := enc.Encode(hashes, packWindow)
8585
c.Assert(err, IsNil)
8686

87-
f, err := memfs.New().Create("packfile")
87+
fs := memfs.New()
88+
f, err := fs.Create("packfile")
8889
c.Assert(err, IsNil)
8990

9091
_, err = f.Write(buf.Bytes())
@@ -105,7 +106,7 @@ func (s *EncoderAdvancedSuite) testEncodeDecode(
105106
_, err = f.Seek(0, io.SeekStart)
106107
c.Assert(err, IsNil)
107108

108-
p := NewPackfile(index, f)
109+
p := NewPackfile(index, fs, f)
109110

110111
decodeHash, err := p.ID()
111112
c.Assert(err, IsNil)

‎plumbing/format/packfile/encoder_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"io"
66
stdioutil "io/ioutil"
77

8-
"gopkg.in/src-d/go-billy.v3/memfs"
8+
"gopkg.in/src-d/go-billy.v4/memfs"
99
"gopkg.in/src-d/go-git.v4/plumbing"
1010
"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
1111
"gopkg.in/src-d/go-git.v4/storage/memory"
@@ -290,7 +290,8 @@ func objectsEqual(c *C, o1, o2 plumbing.EncodedObject) {
290290
}
291291

292292
func packfileFromReader(c *C, buf *bytes.Buffer) (*Packfile, func()) {
293-
file, err := memfs.New().Create("packfile")
293+
fs := memfs.New()
294+
file, err := fs.Create("packfile")
294295
c.Assert(err, IsNil)
295296

296297
_, err = file.Write(buf.Bytes())
@@ -311,7 +312,7 @@ func packfileFromReader(c *C, buf *bytes.Buffer) (*Packfile, func()) {
311312
index, err := w.Index()
312313
c.Assert(err, IsNil)
313314

314-
return NewPackfile(index, file), func() {
315+
return NewPackfile(index, fs, file), func() {
315316
c.Assert(file.Close(), IsNil)
316317
}
317318
}

‎plumbing/format/packfile/fsobject.go‎

Lines changed: 55 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,23 @@ package packfile
33
import (
44
"io"
55

6+
billy "gopkg.in/src-d/go-billy.v4"
67
"gopkg.in/src-d/go-git.v4/plumbing"
8+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
9+
"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
710
)
811

912
// FSObject is an object from the packfile on the filesystem.
1013
type FSObject struct {
11-
hash plumbing.Hash
12-
h *ObjectHeader
13-
offset int64
14-
size int64
15-
typ plumbing.ObjectType
16-
packfile *Packfile
14+
hash plumbing.Hash
15+
h *ObjectHeader
16+
offset int64
17+
size int64
18+
typ plumbing.ObjectType
19+
index idxfile.Index
20+
fs billy.Filesystem
21+
path string
22+
cache cache.Object
1723
}
1824

1925
// NewFSObject creates a new filesystem object.
@@ -22,20 +28,42 @@ func NewFSObject(
2228
finalType plumbing.ObjectType,
2329
offset int64,
2430
contentSize int64,
25-
packfile *Packfile,
31+
index idxfile.Index,
32+
fs billy.Filesystem,
33+
path string,
34+
cache cache.Object,
2635
) *FSObject {
2736
return &FSObject{
28-
hash: hash,
29-
offset: offset,
30-
size: contentSize,
31-
typ: finalType,
32-
packfile: packfile,
37+
hash: hash,
38+
offset: offset,
39+
size: contentSize,
40+
typ: finalType,
41+
index: index,
42+
fs: fs,
43+
path: path,
44+
cache: cache,
3345
}
3446
}
3547

3648
// Reader implements the plumbing.EncodedObject interface.
3749
func (o *FSObject) Reader() (io.ReadCloser, error) {
38-
return o.packfile.getObjectContent(o.offset)
50+
f, err := o.fs.Open(o.path)
51+
if err != nil {
52+
return nil, err
53+
}
54+
55+
p := NewPackfileWithCache(o.index, nil, f, o.cache)
56+
r, err := p.getObjectContent(o.offset)
57+
if err != nil {
58+
_ = f.Close()
59+
return nil, err
60+
}
61+
62+
if err := f.Close(); err != nil {
63+
return nil, err
64+
}
65+
66+
return r, nil
3967
}
4068

4169
// SetSize implements the plumbing.EncodedObject interface. This method
@@ -62,3 +90,17 @@ func (o *FSObject) Type() plumbing.ObjectType {
6290
func (o *FSObject) Writer() (io.WriteCloser, error) {
6391
return nil, nil
6492
}
93+
94+
type objectReader struct {
95+
io.ReadCloser
96+
f billy.File
97+
}
98+
99+
func (r *objectReader) Close() error {
100+
if err := r.ReadCloser.Close(); err != nil {
101+
_ = r.f.Close()
102+
return err
103+
}
104+
105+
return r.f.Close()
106+
}

‎plumbing/format/packfile/packfile.go‎

Lines changed: 39 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ package packfile
33
import (
44
"bytes"
55
"io"
6-
stdioutil "io/ioutil"
76
"os"
87

8+
billy "gopkg.in/src-d/go-billy.v4"
99
"gopkg.in/src-d/go-git.v4/plumbing"
1010
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1111
"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
@@ -24,21 +24,26 @@ var (
2424
// Packfile allows retrieving information from inside a packfile.
2525
type Packfile struct {
2626
idxfile.Index
27-
file io.ReadSeeker
27+
fs billy.Filesystem
28+
file billy.File
2829
s *Scanner
2930
deltaBaseCache cache.Object
3031
offsetToType map[int64]plumbing.ObjectType
3132
}
3233

3334
// NewPackfileWithCache creates a new Packfile with the given object cache.
35+
// If the filesystem is provided, the packfile will return FSObjects, otherwise
36+
// it will return MemoryObjects.
3437
func NewPackfileWithCache(
3538
index idxfile.Index,
36-
file io.ReadSeeker,
39+
fs billy.Filesystem,
40+
file billy.File,
3741
cache cache.Object,
3842
) *Packfile {
3943
s := NewScanner(file)
4044
return &Packfile{
4145
index,
46+
fs,
4247
file,
4348
s,
4449
cache,
@@ -48,8 +53,10 @@ func NewPackfileWithCache(
4853

4954
// NewPackfile returns a packfile representation for the given packfile file
5055
// and packfile idx.
51-
func NewPackfile(index idxfile.Index, file io.ReadSeeker) *Packfile {
52-
return NewPackfileWithCache(index, file, cache.NewObjectLRUDefault())
56+
// If the filesystem is provided, the packfile will return FSObjects, otherwise
57+
// it will return MemoryObjects.
58+
func NewPackfile(index idxfile.Index, fs billy.Filesystem, file billy.File) *Packfile {
59+
return NewPackfileWithCache(index, fs, file, cache.NewObjectLRUDefault())
5360
}
5461

5562
// Get retrieves the encoded object in the packfile with the given hash.
@@ -215,6 +222,12 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) {
215222
return nil, err
216223
}
217224

225+
// If we have no filesystem, we will return a MemoryObject instead
226+
// of an FSObject.
227+
if p.fs == nil {
228+
return p.getNextObject(h)
229+
}
230+
218231
hash, err := p.FindHash(h.Offset)
219232
if err != nil {
220233
return nil, err
@@ -232,7 +245,16 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) {
232245

233246
p.offsetToType[h.Offset] = typ
234247

235-
return NewFSObject(hash, typ, h.Offset, size, p), nil
248+
return NewFSObject(
249+
hash,
250+
typ,
251+
h.Offset,
252+
size,
253+
p.Index,
254+
p.fs,
255+
p.file.Name(),
256+
p.deltaBaseCache,
257+
), nil
236258
}
237259

238260
func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) {
@@ -245,10 +267,20 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) {
245267
return nil, err
246268
}
247269

270+
obj, err := p.getNextObject(h)
271+
if err != nil {
272+
return nil, err
273+
}
274+
275+
return obj.Reader()
276+
}
277+
278+
func (p *Packfile) getNextObject(h *ObjectHeader) (plumbing.EncodedObject, error) {
248279
var obj = new(plumbing.MemoryObject)
249280
obj.SetSize(h.Length)
250281
obj.SetType(h.Type)
251282

283+
var err error
252284
switch h.Type {
253285
case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject:
254286
err = p.fillRegularObjectContent(obj)
@@ -264,7 +296,7 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) {
264296
return nil, err
265297
}
266298

267-
return obj.Reader()
299+
return obj, nil
268300
}
269301

270302
func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) error {
@@ -410,29 +442,6 @@ func (p *Packfile) Close() error {
410442
return closer.Close()
411443
}
412444

413-
// MemoryObjectFromDisk converts a FSObject to a MemoryObject.
414-
func MemoryObjectFromDisk(obj plumbing.EncodedObject) (plumbing.EncodedObject, error) {
415-
o2 := new(plumbing.MemoryObject)
416-
o2.SetType(obj.Type())
417-
o2.SetSize(obj.Size())
418-
419-
r, err := obj.Reader()
420-
if err != nil {
421-
return nil, err
422-
}
423-
424-
data, err := stdioutil.ReadAll(r)
425-
if err != nil {
426-
return nil, err
427-
}
428-
429-
if _, err := o2.Write(data); err != nil {
430-
return nil, err
431-
}
432-
433-
return o2, nil
434-
}
435-
436445
type objectIter struct {
437446
p *Packfile
438447
typ plumbing.ObjectType

‎plumbing/format/packfile/packfile_test.go‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -109,13 +109,14 @@ var expectedEntries = map[plumbing.Hash]int64{
109109
func (s *PackfileSuite) SetUpTest(c *C) {
110110
s.f = fixtures.Basic().One()
111111

112-
f, err := osfs.New("").Open(s.f.Packfile().Name())
112+
fs := osfs.New("")
113+
f, err := fs.Open(s.f.Packfile().Name())
113114
c.Assert(err, IsNil)
114115

115116
s.idx = idxfile.NewMemoryIndex()
116117
c.Assert(idxfile.NewDecoder(s.f.Idx()).Decode(s.idx), IsNil)
117118

118-
s.p = packfile.NewPackfile(s.idx, f)
119+
s.p = packfile.NewPackfile(s.idx, fs, f)
119120
}
120121

121122
func (s *PackfileSuite) TearDownTest(c *C) {
@@ -125,7 +126,11 @@ func (s *PackfileSuite) TearDownTest(c *C) {
125126
func (s *PackfileSuite) TestDecode(c *C) {
126127
fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) {
127128
index := getIndexFromIdxFile(f.Idx())
128-
p := packfile.NewPackfile(index, f.Packfile())
129+
fs := osfs.New("")
130+
pf, err := fs.Open(f.Packfile().Name())
131+
c.Assert(err, IsNil)
132+
133+
p := packfile.NewPackfile(index, fs, pf)
129134
defer p.Close()
130135

131136
for _, h := range expectedHashes {
@@ -140,7 +145,11 @@ func (s *PackfileSuite) TestDecodeByTypeRefDelta(c *C) {
140145
f := fixtures.Basic().ByTag("ref-delta").One()
141146

142147
index := getIndexFromIdxFile(f.Idx())
143-
packfile := packfile.NewPackfile(index, f.Packfile())
148+
fs := osfs.New("")
149+
pf, err := fs.Open(f.Packfile().Name())
150+
c.Assert(err, IsNil)
151+
152+
packfile := packfile.NewPackfile(index, fs, pf)
144153
defer packfile.Close()
145154

146155
iter, err := packfile.GetByType(plumbing.CommitObject)
@@ -171,7 +180,11 @@ func (s *PackfileSuite) TestDecodeByType(c *C) {
171180
fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) {
172181
for _, t := range ts {
173182
index := getIndexFromIdxFile(f.Idx())
174-
packfile := packfile.NewPackfile(index, f.Packfile())
183+
fs := osfs.New("")
184+
pf, err := fs.Open(f.Packfile().Name())
185+
c.Assert(err, IsNil)
186+
187+
packfile := packfile.NewPackfile(index, fs, pf)
175188
defer packfile.Close()
176189

177190
iter, err := packfile.GetByType(t)
@@ -188,10 +201,14 @@ func (s *PackfileSuite) TestDecodeByType(c *C) {
188201
func (s *PackfileSuite) TestDecodeByTypeConstructor(c *C) {
189202
f := fixtures.Basic().ByTag("packfile").One()
190203
index := getIndexFromIdxFile(f.Idx())
191-
packfile := packfile.NewPackfile(index, f.Packfile())
204+
fs := osfs.New("")
205+
pf, err := fs.Open(f.Packfile().Name())
206+
c.Assert(err, IsNil)
207+
208+
packfile := packfile.NewPackfile(index, fs, pf)
192209
defer packfile.Close()
193210

194-
_, err := packfile.GetByType(plumbing.OFSDeltaObject)
211+
_, err = packfile.GetByType(plumbing.OFSDeltaObject)
195212
c.Assert(err, Equals, plumbing.ErrInvalidType)
196213

197214
_, err = packfile.GetByType(plumbing.REFDeltaObject)

‎plumbing/object/blob_test.go‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"io/ioutil"
77

88
"gopkg.in/src-d/go-git.v4/plumbing"
9-
"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
109

1110
. "gopkg.in/check.v1"
1211
)
@@ -71,12 +70,6 @@ func (s *BlobsSuite) TestBlobIter(c *C) {
7170

7271
blobs := []*Blob{}
7372
iter.ForEach(func(b *Blob) error {
74-
var err error
75-
b.obj, err = packfile.MemoryObjectFromDisk(b.obj)
76-
if err != nil {
77-
return err
78-
}
79-
8073
blobs = append(blobs, b)
8174
return nil
8275
})

0 commit comments

Comments
 (0)