From eef48c4be357c2f888f7c627eeb123666de6cde8 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 3 Mar 2020 18:31:35 -0800 Subject: [PATCH] feat(client): support car to filestore Store filestore reference for CAR files --- go.mod | 6 ++-- go.sum | 9 ++++-- node/impl/client/client.go | 60 ++++++++++++++++++++++++++------------ 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index fe67193e5..7acee6f9c 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c - github.com/ipfs/go-car v0.0.3-0.20200221191037-3762780fa84e + github.com/ipfs/go-car v0.0.3-0.20200304012825-b6769248bfef github.com/ipfs/go-cid v0.0.5 github.com/ipfs/go-datastore v0.4.2 github.com/ipfs/go-ds-badger2 v0.0.0-20200211201106-609c9d2a39c7 @@ -45,9 +45,9 @@ require ( github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242 - github.com/ipfs/go-ipfs-blockstore v0.1.3 + github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.1 - github.com/ipfs/go-ipfs-ds-help v0.0.1 + github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.4 diff --git a/go.sum b/go.sum index d68646204..45317d78d 100644 --- a/go.sum +++ b/go.sum @@ -229,8 +229,8 @@ github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7s github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-car v0.0.3-0.20200131220434-3f68f6ebd093/go.mod h1:rEkw0S1sHd5kHL3rUSGEhwNanYqTwwNhjtpp0rwjrr4= -github.com/ipfs/go-car v0.0.3-0.20200221191037-3762780fa84e h1:+wPt/tQHMOl00ekuf+5+ekWpyGQ6JOnF9vzEXjW7LtA= -github.com/ipfs/go-car v0.0.3-0.20200221191037-3762780fa84e/go.mod h1:rEkw0S1sHd5kHL3rUSGEhwNanYqTwwNhjtpp0rwjrr4= +github.com/ipfs/go-car v0.0.3-0.20200304012825-b6769248bfef h1:Zn2PZSkX8Go+SZpQmjVKNrkcgbNuIxUC/3MOQRDTIVw= +github.com/ipfs/go-car v0.0.3-0.20200304012825-b6769248bfef/go.mod h1:7BMxYRi5cbR/GJ1A8mYSHvMLXLkHgYdrJ6VlNGobd0o= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -266,8 +266,9 @@ github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242 h1:OYVGeYkGSR github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242/go.mod h1:kq3Pi+UP3oHhAdKexE+kHHYRKMoFNuGero0R7q3hWGg= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= -github.com/ipfs/go-ipfs-blockstore v0.1.3 h1:Nc/Uwc8KZgo5PsMUZT/Zt7zc0u/s+b/3c64ChQNttHc= github.com/ipfs/go-ipfs-blockstore v0.1.3/go.mod h1:iNWVBoSQ7eMcaGo8+L3pKZABGTdWcqj1/hpoUu5bDps= +github.com/ipfs/go-ipfs-blockstore v0.1.4 h1:2SGI6U1B44aODevza8Rde3+dY30Pb+lbcObe1LETxOQ= +github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= @@ -277,6 +278,8 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v0.1.1 h1:IW/bXGeaAZV2VH0Kuok+Ohva/zHkHmeLFBxC1k7mNPc= +github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a3afa7a81..f84a80576 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -201,8 +201,23 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error return cid.Undef, err } + stat, err := f.Stat() + if err != nil { + return cid.Undef, err + } + + file, err := files.NewReaderPathFile(ref.Path, f, stat) + if err != nil { + return cid.Undef, err + } if ref.IsCAR { - result, err := car.LoadCar(a.Blockstore, f) + var store car.Store + if a.Filestore == nil { + store = a.Blockstore + } else { + store = (*filestore.Filestore)(a.Filestore) + } + result, err := car.LoadCar(store, file) if err != nil { return cid.Undef, err } @@ -214,16 +229,6 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error return result.Roots[0], nil } - stat, err := f.Stat() - if err != nil { - return cid.Undef, err - } - - file, err := files.NewReaderPathFile(ref.Path, f, stat) - if err != nil { - return cid.Undef, err - } - bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG) params := ihelper.DagBuilderParams{ @@ -286,20 +291,37 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) { // TODO: make this less very bad by tracking root cids instead of using ListAll out := make([]api.Import, 0) + lowest := make([]uint64, 0) for { r := next() if r == nil { return out, nil } - if r.Offset != 0 { - continue + matched := false + for i := range out { + if out[i].FilePath == r.FilePath { + matched = true + if lowest[i] > r.Offset { + lowest[i] = r.Offset + out[i] = api.Import{ + Status: r.Status, + Key: r.Key, + FilePath: r.FilePath, + Size: r.Size, + } + } + break + } + } + if !matched { + out = append(out, api.Import{ + Status: r.Status, + Key: r.Key, + FilePath: r.FilePath, + Size: r.Size, + }) + lowest = append(lowest, r.Offset) } - out = append(out, api.Import{ - Status: r.Status, - Key: r.Key, - FilePath: r.FilePath, - Size: r.Size, - }) } }