From 662c51605fc59a7998aa29b80d9399d6e3af6e68 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 10 Sep 2021 15:12:43 +0200 Subject: [PATCH] feat: retrieve from any cid (not just root cid) --- go.mod | 16 ++-- go.sum | 36 +++++---- itests/deals_anycid_test.go | 143 ++++++++++++++++++++++++++++++++++++ itests/kit/deals.go | 11 ++- itests/kit/files.go | 25 +++++-- markets/dagstore/wrapper.go | 24 ++++++ 6 files changed, 222 insertions(+), 33 deletions(-) create mode 100644 itests/deals_anycid_test.go diff --git a/go.mod b/go.mod index 37e6bb916..daeb3847d 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/elastic/gosigar v0.14.1 github.com/etclabscore/go-openrpc-reflect v0.0.36 github.com/fatih/color v1.13.0 - github.com/filecoin-project/dagstore v0.4.3 + github.com/filecoin-project/dagstore v0.4.4-0.20210915102524-cd372c4b14d5 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f github.com/filecoin-project/go-address v0.0.6 github.com/filecoin-project/go-bitfield v0.2.4 @@ -36,11 +36,11 @@ require ( github.com/filecoin-project/go-data-transfer v1.11.4 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.13.3 - github.com/filecoin-project/go-jsonrpc v0.1.5 - github.com/filecoin-project/go-padreader v0.0.1 - github.com/filecoin-project/go-paramfetch v0.0.2 - github.com/filecoin-project/go-state-types v0.1.1 + github.com/filecoin-project/go-fil-markets v1.12.1-0.20210916145046-d81117f778de + github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec + github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 + github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498 + github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e github.com/filecoin-project/go-statemachine v1.0.1 github.com/filecoin-project/go-statestore v0.1.1 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b @@ -98,7 +98,7 @@ require ( github.com/ipfs/go-unixfs v0.2.6 github.com/ipfs/interface-go-ipfs-core v0.4.0 github.com/ipld/go-car v0.3.2-0.20211001225732-32d0d9933823 - github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7 + github.com/ipld/go-car/v2 v2.0.3-0.20210910134712-2d2568297f26 github.com/ipld/go-codec-dagpb v1.3.0 github.com/ipld/go-ipld-prime v0.12.3 github.com/ipld/go-ipld-selector-text-lite v0.0.0 @@ -139,7 +139,7 @@ require ( github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 - github.com/urfave/cli/v2 v2.2.0 + github.com/urfave/cli/v2 v2.3.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8 github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 diff --git a/go.sum b/go.sum index 6019140fc..0eb3c7d05 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,8 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/filecoin-project/dagstore v0.4.2/go.mod h1:WY5OoLfnwISCk6eASSF927KKPqLPIlTwmG1qHpA08KY= -github.com/filecoin-project/dagstore v0.4.3 h1:yeFl6+2BRY1gOVp/hrZuFa24s7LY0Qqkqx/Gh8lidZs= -github.com/filecoin-project/dagstore v0.4.3/go.mod h1:dm/91AO5UaDd3bABFjg/5fmRH99vvpS7g1mykqvz6KQ= +github.com/filecoin-project/dagstore v0.4.4-0.20210915102524-cd372c4b14d5 h1:wtKBSxmV5ohSeZHS86TrLtw6m1dO88QaNo6az/QRrYU= +github.com/filecoin-project/dagstore v0.4.4-0.20210915102524-cd372c4b14d5/go.mod h1:dsWYxyEYa5aB62UXJcQhwlIaBOS+kkrKQgYGxwAnmkE= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-address v0.0.6 h1:DWQtj38ax+ogHwyH3VULRIoT8E6loyXqsk/p81xoY7M= @@ -327,6 +327,7 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o= github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= +github.com/filecoin-project/go-data-transfer v1.10.0/go.mod h1:uQtqy6vUAY5v70ZHdkF5mJ8CjVtjj/JA3aOoaqzWTVw= github.com/filecoin-project/go-data-transfer v1.11.4 h1:jKvlx0/C8HSyLRn/G1P9TjtfBtFU9jbCvCVFmWbyYVQ= github.com/filecoin-project/go-data-transfer v1.11.4/go.mod h1:2MitLI0ebCkLlPKM7NRggP/t9d+gCcREUKkCKqWRCwU= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= @@ -338,8 +339,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.13.3 h1:iMCpG7I4fb+YLcgDnMaqZiZiyFZWNvrwHqiFPHB0/tQ= -github.com/filecoin-project/go-fil-markets v1.13.3/go.mod h1:38zuj8AgDvOfdakFLpC/syYIYgXTzkq7xqBJ6T1AuG4= +github.com/filecoin-project/go-fil-markets v1.12.1-0.20210916145046-d81117f778de h1:QTcLAfS3DAWtWvwCG49wjN8jX0sJArnwOyyKQwbvBmU= +github.com/filecoin-project/go-fil-markets v1.12.1-0.20210916145046-d81117f778de/go.mod h1:XuuZFaFujI47nrgfQJiq7jWB+6rRya6nm7Sj6uXQ80U= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -347,24 +348,22 @@ github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+ github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGyDjJjYSRX7hp/FGOStdqrWyDI= github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 h1:rVVNq0x6RGQIzCo1iiJlGFm9AGIZzeifggxtKMU7zmI= github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g= -github.com/filecoin-project/go-jsonrpc v0.1.5 h1:ckxqZ09ivBAVf5CSmxxrqqNHC7PJm3GYGtYKiNQ+vGk= -github.com/filecoin-project/go-jsonrpc v0.1.5/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec h1:rGI5I7fdU4viManxmDdbk5deZO7afe6L1Wc04dAmlOM= +github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= +github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 h1:0BogtftbcgyBx4lP2JWM00ZK7/pXmgnrDqKp9aLTgVs= github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= -github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs= -github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= -github.com/filecoin-project/go-paramfetch v0.0.2 h1:a6W3Ij6CKhwHYYlx+5mqvBIyw4CabZH2ojdEaoAZ6/g= -github.com/filecoin-project/go-paramfetch v0.0.2/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498 h1:G10ezOvpH1CLXQ19EA9VWNwyL0mg536ujSayjV0yg0k= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20210614165157-25a6c7769498/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e h1:XAgb6HmgXaGRklNjhZoNMSIYriKLqjWXIqYMotg6iSs= github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.1.1 h1:LR260vya4p++atgf256W6yV3Lxl5mKrBFcEZePWQrdg= -github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v1.0.1 h1:LQ60+JDVjMdLxXmVFM2jjontzOYnfVE7u02CXV3WKSw= github.com/filecoin-project/go-statemachine v1.0.1/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= @@ -742,6 +741,7 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= +github.com/ipfs/go-graphsync v0.9.0/go.mod h1:J62ahWT9JbPsFL2UWsUM5rOu0lZJ0LOIH1chHdxGGcw= github.com/ipfs/go-graphsync v0.10.0/go.mod h1:cKIshzTaa5rCZjryH5xmSKZVGX9uk1wvwGvz2WEha5Y= github.com/ipfs/go-graphsync v0.10.4 h1:1WZhyOPxgxLvHTIC2GoLltaBrjZ+JuXC2oKAEiX8f3Y= github.com/ipfs/go-graphsync v0.10.4/go.mod h1:oei4tnWAKnZ6LPnapZGPYVVbyiKV1UP3f8BeLU7Z4JQ= @@ -861,12 +861,13 @@ github.com/ipfs/iptb-plugins v0.3.0/go.mod h1:5QtOvckeIw4bY86gSH4fgh3p3gCSMn3FmI github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw= github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ= +github.com/ipld/go-car v0.3.1-0.20210601190600-f512dac51e8e/go.mod h1:wUxBdwOLA9/0HZBi3fnTBzla0MuwlqgJLyrhOg1XaKI= github.com/ipld/go-car v0.3.2-0.20211001225732-32d0d9933823 h1:8JMSJ0k71fU9lIUrpVwEdoX4KoxiTEX8cZG97v/hTDw= github.com/ipld/go-car v0.3.2-0.20211001225732-32d0d9933823/go.mod h1:jSlTph+i/q1jLFoiKKeN69KGG0fXpwrcD0izu5C1Tpo= github.com/ipld/go-car/v2 v2.0.0-beta1.0.20210721090610-5a9d1b217d25/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= github.com/ipld/go-car/v2 v2.0.2/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= -github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7 h1:6Z0beJSZNsRY+7udoqUl4gQ/tqtrPuRvDySrlsvbqZA= -github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= +github.com/ipld/go-car/v2 v2.0.3-0.20210910134712-2d2568297f26 h1:IBXjeOANJGYZVJ9jsvf0Tu+ANQ9FLstscJgzfJgwA8c= +github.com/ipld/go-car/v2 v2.0.3-0.20210910134712-2d2568297f26/go.mod h1:bBmZNBj4jfOHrd4Cu/koFhqwsCnZ5RrGXU1wwWqDr20= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= @@ -878,6 +879,7 @@ github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/j github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= +github.com/ipld/go-ipld-prime v0.12.0/go.mod h1:hy8b93WleDMRKumOJnTIrr0MbbFbx9GD6Kzxa53Xppc= github.com/ipld/go-ipld-prime v0.12.3-0.20210930132912-0b3aef3ca569/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= github.com/ipld/go-ipld-prime v0.12.3 h1:furVobw7UBLQZwlEwfE26tYORy3PAK8VYSgZOSr3JMQ= github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= @@ -1482,8 +1484,9 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.2.1-0.20210714093213-b2b5bd6fe68b/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.3.0 h1:tstDwfIjiHbnIjeM5Lp+pMrSeN+LCMsEwOrkPmWm03A= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= +github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 h1:ZrUuMKNgJ52qHPoQ+bx0h0uBfcWmN7Px+4uKSZeesiI= +github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1794,8 +1797,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= diff --git a/itests/deals_anycid_test.go b/itests/deals_anycid_test.go new file mode 100644 index 000000000..5060847b9 --- /dev/null +++ b/itests/deals_anycid_test.go @@ -0,0 +1,143 @@ +package itests + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/filecoin-project/lotus/api" + + ipld "github.com/ipfs/go-ipld-format" + + "github.com/filecoin-project/go-fil-markets/shared" + + "github.com/ipld/go-car" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/lotus/node/modules/dtypes" + + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/modules" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/itests/kit" + logging "github.com/ipfs/go-log/v2" +) + +func TestDealRetrieveByAnyCid(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + + ctx := context.Background() + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) + + // Allow 8MB sectors + kit.EnableLargeSectors(t) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) + kit.QuietMiningLogs() + //logging.SetLogLevel("dagstore.invidx", "debug") + logging.SetLogLevel("markets-rtvl", "debug") + logging.SetLogLevel("markets-rtvl-reval", "debug") + + // For these tests where the block time is artificially short, just use + // a deal start epoch that is guaranteed to be far enough in the future + // so that the deal starts sealing in time + startEpoch := abi.ChainEpoch(2 << 12) + + // Override the dependency injection for the blockstore accessor, so that + // we can get a reference to the blockstore containing our deal later in + // the test + var bsa storagemarket.BlockstoreAccessor + bsaFn := func(importmgr dtypes.ClientImportMgr) storagemarket.BlockstoreAccessor { + bsa = modules.StorageBlockstoreAccessor(importmgr) + return bsa + } + bsaOpt := kit.ConstructorOpts(node.Override(new(storagemarket.BlockstoreAccessor), bsaFn)) + + // Allow 512MB sectors + eightMBSectorsOpt := kit.SectorSize(8 << 20) + + // Create a client, and a miner with its own full node + _, client, miner, ens := kit.EnsembleTwoOne(t, kit.MockProofs(), bsaOpt, eightMBSectorsOpt) + ens.InterconnectAll().BeginMining(250 * time.Millisecond) + + dh := kit.NewDealHarness(t, client, miner, miner) + + //res, path := client.CreateImportFile(ctx, 5, 140*1024*1024) + dagOpts := kit.GeneratedDAGOpts{ + ChunkSize: 1024, + Maxlinks: 2, + } + carv1FilePath, _ := kit.CreateRandomCARv1(t, 5, 8*1024, dagOpts) + res, err := client.ClientImport(ctx, api.FileRef{Path: carv1FilePath, IsCAR: true}) + require.NoError(t, err) + + // Get the blockstore for the file + bs, err := bsa.Get(res.Root) + require.NoError(t, err) + + // Get all CIDs from the file + sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: res.Root, Selector: shared.AllSelector()}}) + prepared, err := sc.Prepare() + require.NoError(t, err) + cids := prepared.Cids() + // fmt.Println(cids) + for i, c := range cids { + blk, err := bs.Get(c) + require.NoError(t, err) + + nd, err := ipld.Decode(blk) + require.NoError(t, err) + + fmt.Println(i, c, len(nd.Links())) + + if i > 100 { + break + } + } + + // Create a storage deal + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.DealStartEpoch = startEpoch + dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price + dealCid := dh.StartDeal(ctx, dp) + + // Wait for the deal to be sealed + dh.WaitDealSealed(ctx, dealCid, false, false, nil) + + ask, err := miner.MarketGetRetrievalAsk(ctx) + require.NoError(t, err) + ask.PricePerByte = abi.NewTokenAmount(0) + ask.UnsealPrice = abi.NewTokenAmount(0) + err = miner.MarketSetRetrievalAsk(ctx, ask) + require.NoError(t, err) + + // Fetch the deal data + info, err := client.ClientGetDealInfo(ctx, *dealCid) + require.NoError(t, err) + + //targetCid := res.Root + targetCid := cids[1] + offer, err := client.ClientMinerQueryOffer(ctx, miner.ActorAddr, targetCid, &info.PieceCID) + require.NoError(t, err) + require.Empty(t, offer.Err) + + outPath := dh.PerformRetrievalForOffer(ctx, false, offer) + stat, err := os.Stat(outPath) + require.NoError(t, err) + fmt.Println(stat) + + // TODO: compare stored vs retrieved blocks + //kit.AssertFilesEqual(t, path, outPath) +} diff --git a/itests/kit/deals.go b/itests/kit/deals.go index 4a9af69e6..3a7d941ff 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -303,15 +303,19 @@ func (dh *DealHarness) StartSealingWaiting(ctx context.Context) { } } -func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root cid.Cid, carExport bool) (path string) { +func (dh *DealHarness) PerformRetrieval(ctx context.Context, dealCid *cid.Cid, root cid.Cid, carExport bool) (path string) { // perform retrieval. - info, err := dh.client.ClientGetDealInfo(ctx, *deal) + info, err := dh.client.ClientGetDealInfo(ctx, *dealCid) require.NoError(dh.t, err) offers, err := dh.client.ClientFindData(ctx, root, &info.PieceCID) require.NoError(dh.t, err) require.NotEmpty(dh.t, offers, "no offers") + return dh.PerformRetrievalForOffer(ctx, carExport, offers[0]) +} + +func (dh *DealHarness) PerformRetrievalForOffer(ctx context.Context, carExport bool, offer api.QueryOffer) string { carFile, err := ioutil.TempFile(dh.t.TempDir(), "ret-car") require.NoError(dh.t, err) @@ -325,7 +329,8 @@ func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root IsCAR: carExport, } - updates, err := dh.client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref) + order := offer.Order(caddr) + updates, err := dh.client.ClientRetrieveWithEvents(ctx, order, ref) require.NoError(dh.t, err) for update := range updates { diff --git a/itests/kit/files.go b/itests/kit/files.go index 9babac941..d69c6d776 100644 --- a/itests/kit/files.go +++ b/itests/kit/files.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/require" ) -const unixfsChunkSize uint64 = 1 << 10 +const unixfsChunkSize int64 = 1 << 10 var defaultHashFunction = uint64(mh.BLAKE2B_MIN + 31) @@ -54,7 +54,7 @@ func CreateRandomFile(t *testing.T, rseed, size int) (path string) { // CreateRandomFile creates a normal file with the provided seed and the // provided size and then transforms it to a CARv1 file and returns it. -func CreateRandomCARv1(t *testing.T, rseed, size int) (carV1FilePath string, origFilePath string) { +func CreateRandomCARv1(t *testing.T, rseed, size int, opts ...GeneratedDAGOpts) (carV1FilePath string, origFilePath string) { ctx := context.Background() if size == 0 { size = 1600 @@ -75,7 +75,7 @@ func CreateRandomCARv1(t *testing.T, rseed, size int) (carV1FilePath string, ori bs := bstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) dagSvc := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - root := writeUnixfsDAG(ctx, t, file, dagSvc) + root := writeUnixfsDAG(ctx, t, file, dagSvc, opts...) // create a CARv1 file from the DAG tmp, err := os.CreateTemp(t.TempDir(), "randcarv1") @@ -92,7 +92,20 @@ func CreateRandomCARv1(t *testing.T, rseed, size int) (carV1FilePath string, ori return tmp.Name(), file.Name() } -func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldformat.DAGService) cid.Cid { +type GeneratedDAGOpts struct { + ChunkSize int64 + Maxlinks int +} + +func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldformat.DAGService, opts ...GeneratedDAGOpts) cid.Cid { + dagOpts := GeneratedDAGOpts{ + ChunkSize: unixfsChunkSize, + Maxlinks: 1024, + } + if len(opts) > 0 { + dagOpts = opts[0] + } + rpf := files.NewReaderFile(rd) // generate the dag and get the root @@ -103,7 +116,7 @@ func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldfor bufferedDS := ipldformat.NewBufferedDAG(ctx, dag) params := ihelper.DagBuilderParams{ - Maxlinks: 1024, + Maxlinks: dagOpts.Maxlinks, RawLeaves: true, CidBuilder: cidutil.InlineBuilder{ Builder: prefix, @@ -112,7 +125,7 @@ func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldfor Dagserv: bufferedDS, } - db, err := params.New(chunk.NewSizeSplitter(rpf, int64(unixfsChunkSize))) + db, err := params.New(chunk.NewSizeSplitter(rpf, dagOpts.ChunkSize)) require.NoError(t, err) nd, err := balanced.Layout(db) diff --git a/markets/dagstore/wrapper.go b/markets/dagstore/wrapper.go index 3d9288659..6a32c011a 100644 --- a/markets/dagstore/wrapper.go +++ b/markets/dagstore/wrapper.go @@ -3,6 +3,7 @@ package dagstore import ( "context" "errors" + "fmt" "math" "os" "path/filepath" @@ -398,6 +399,29 @@ func (w *Wrapper) markRegistrationComplete() error { return file.Close() } +// Get all the pieces that contain a block +func (w *Wrapper) GetPiecesContainingBlock(blockCID cid.Cid) ([]cid.Cid, error) { + // Pieces are stored as "shards" in the DAG store + shardKeys, err := w.dagst.GetShardKeysForCid(blockCID) + if err != nil { + return nil, xerrors.Errorf("getting pieces containing block %s: %w", blockCID, err) + } + + // Convert from shard key to cid + pieceCids := make([]cid.Cid, 0, len(shardKeys)) + for _, k := range shardKeys { + c, err := cid.Parse(k.String()) + if err != nil { + prefix := fmt.Sprintf("getting pieces containing block %s:", blockCID) + return nil, xerrors.Errorf(prefix+" converting shard key %s to piece cid: %w", k, err) + } + + pieceCids = append(pieceCids, c) + } + + return pieceCids, nil +} + func (w *Wrapper) Close() error { // Cancel the context w.cancel()