remove option for publishing through ipfs node interface

This commit is contained in:
Ian Norden
2020-08-12 09:26:18 -05:00
parent 39354b2114
commit a532e17e21
55 changed files with 206 additions and 2311 deletions
-90
View File
@@ -1,90 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"context"
"github.com/sirupsen/logrus"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/plugin/loader"
"github.com/ipfs/go-ipfs/repo/fsrepo"
ipld "github.com/ipfs/go-ipld-format"
)
// InitIPFSPlugins is used to initialized IPFS plugins before creating a new IPFS node
// This should only be called once
func InitIPFSPlugins() error {
logrus.Debug("initializing IPFS plugins")
l, err := loader.NewPluginLoader("")
if err != nil {
return err
}
err = l.Initialize()
if err != nil {
return err
}
return l.Inject()
}
// InitIPFSBlockService is used to configure and return a BlockService using an ipfs repo path (e.g. ~/.ipfs)
func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
logrus.Debug("initializing IPFS block service interface")
r, openErr := fsrepo.Open(ipfsPath)
if openErr != nil {
return nil, openErr
}
ctx := context.Background()
cfg := &core.BuildCfg{
Online: false,
Repo: r,
}
ipfsNode, newNodeErr := core.NewNode(ctx, cfg)
if newNodeErr != nil {
return nil, newNodeErr
}
return ipfsNode.Blocks, nil
}
type IPFS struct {
n *core.IpfsNode
ctx context.Context
}
func (ipfs IPFS) Add(node ipld.Node) error {
return ipfs.n.DAG.Add(ipfs.n.Context(), node)
}
func InitIPFSNode(repoPath string) (*IPFS, error) {
logrus.Debug("initializing IPFS node interface")
r, err := fsrepo.Open(repoPath)
if err != nil {
return nil, err
}
ctx := context.Background()
cfg := &core.BuildCfg{
Online: false,
Repo: r,
}
ipfsNode, err := core.NewNode(ctx, cfg)
if err != nil {
return nil, err
}
return &IPFS{n: ipfsNode, ctx: ctx}, nil
}
-50
View File
@@ -1,50 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
var (
duplicateKeyErrorString = "pq: duplicate key value violates unique constraint"
)
type BtcHeaderDagPutter struct {
adder *ipfs.IPFS
}
func NewBtcHeaderDagPutter(adder *ipfs.IPFS) *BtcHeaderDagPutter {
return &BtcHeaderDagPutter{adder: adder}
}
func (bhdp *BtcHeaderDagPutter) DagPut(n node.Node) (string, error) {
header, ok := n.(*ipld.BtcHeader)
if !ok {
return "", fmt.Errorf("BtcHeaderDagPutter expected input type %T got %T", &ipld.BtcHeader{}, n)
}
if err := bhdp.adder.Add(header); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return header.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type BtcTxDagPutter struct {
adder *ipfs.IPFS
}
func NewBtcTxDagPutter(adder *ipfs.IPFS) *BtcTxDagPutter {
return &BtcTxDagPutter{adder: adder}
}
func (etdp *BtcTxDagPutter) DagPut(n node.Node) (string, error) {
transaction, ok := n.(*ipld.BtcTx)
if !ok {
return "", fmt.Errorf("BtcTxDagPutter expected input type %T got %T", &ipld.BtcTx{}, n)
}
if err := etdp.adder.Add(transaction); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return transaction.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type BtcTxTrieDagPutter struct {
adder *ipfs.IPFS
}
func NewBtcTxTrieDagPutter(adder *ipfs.IPFS) *BtcTxTrieDagPutter {
return &BtcTxTrieDagPutter{adder: adder}
}
func (etdp *BtcTxTrieDagPutter) DagPut(n node.Node) (string, error) {
txTrieNode, ok := n.(*ipld.BtcTxTrie)
if !ok {
return "", fmt.Errorf("BtcTxTrieDagPutter expected input type %T got %T", &ipld.BtcTxTrie{}, n)
}
if err := etdp.adder.Add(txTrieNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return txTrieNode.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthHeaderDagPutter struct {
adder *ipfs.IPFS
}
func NewEthBlockHeaderDagPutter(adder *ipfs.IPFS) *EthHeaderDagPutter {
return &EthHeaderDagPutter{adder: adder}
}
func (bhdp *EthHeaderDagPutter) DagPut(n node.Node) (string, error) {
header, ok := n.(*ipld.EthHeader)
if !ok {
return "", fmt.Errorf("EthHeaderDagPutter expected input type %T got %T", &ipld.EthHeader{}, n)
}
if err := bhdp.adder.Add(header); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return header.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthReceiptDagPutter struct {
adder *ipfs.IPFS
}
func NewEthReceiptDagPutter(adder *ipfs.IPFS) *EthReceiptDagPutter {
return &EthReceiptDagPutter{adder: adder}
}
func (erdp *EthReceiptDagPutter) DagPut(n node.Node) (string, error) {
receipt, ok := n.(*ipld.EthReceipt)
if !ok {
return "", fmt.Errorf("EthReceiptDagPutter expected input type %T got type %T", &ipld.EthReceipt{}, n)
}
if err := erdp.adder.Add(receipt); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return receipt.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthRctTrieDagPutter struct {
adder *ipfs.IPFS
}
func NewEthRctTrieDagPutter(adder *ipfs.IPFS) *EthRctTrieDagPutter {
return &EthRctTrieDagPutter{adder: adder}
}
func (etdp *EthRctTrieDagPutter) DagPut(n node.Node) (string, error) {
rctTrieNode, ok := n.(*ipld.EthRctTrie)
if !ok {
return "", fmt.Errorf("EthRctTrieDagPutter expected input type %T got %T", &ipld.EthRctTrie{}, n)
}
if err := etdp.adder.Add(rctTrieNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return rctTrieNode.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthStateDagPutter struct {
adder *ipfs.IPFS
}
func NewEthStateDagPutter(adder *ipfs.IPFS) *EthStateDagPutter {
return &EthStateDagPutter{adder: adder}
}
func (erdp *EthStateDagPutter) DagPut(n node.Node) (string, error) {
stateNode, ok := n.(*ipld.EthStateTrie)
if !ok {
return "", fmt.Errorf("EthStateDagPutter expected input type %T got %T", &ipld.EthStateTrie{}, n)
}
if err := erdp.adder.Add(stateNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return stateNode.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthStorageDagPutter struct {
adder *ipfs.IPFS
}
func NewEthStorageDagPutter(adder *ipfs.IPFS) *EthStorageDagPutter {
return &EthStorageDagPutter{adder: adder}
}
func (erdp *EthStorageDagPutter) DagPut(n node.Node) (string, error) {
storageNode, ok := n.(*ipld.EthStorageTrie)
if !ok {
return "", fmt.Errorf("EthStorageDagPutter expected input type %T got %T", &ipld.EthStorageTrie{}, n)
}
if err := erdp.adder.Add(storageNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return storageNode.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthTxsDagPutter struct {
adder *ipfs.IPFS
}
func NewEthTxsDagPutter(adder *ipfs.IPFS) *EthTxsDagPutter {
return &EthTxsDagPutter{adder: adder}
}
func (etdp *EthTxsDagPutter) DagPut(n node.Node) (string, error) {
transaction, ok := n.(*ipld.EthTx)
if !ok {
return "", fmt.Errorf("EthTxsDagPutter expected input type %T got %T", &ipld.EthTx{}, n)
}
if err := etdp.adder.Add(transaction); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return transaction.Cid().String(), nil
}
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthTxTrieDagPutter struct {
adder *ipfs.IPFS
}
func NewEthTxTrieDagPutter(adder *ipfs.IPFS) *EthTxTrieDagPutter {
return &EthTxTrieDagPutter{adder: adder}
}
func (etdp *EthTxTrieDagPutter) DagPut(n node.Node) (string, error) {
txTrieNode, ok := n.(*ipld.EthTxTrie)
if !ok {
return "", fmt.Errorf("EthTxTrieDagPutter expected input type %T got %T", &ipld.EthTxTrie{}, n)
}
if err := etdp.adder.Add(txTrieNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return txTrieNode.Cid().String(), nil
}
-26
View File
@@ -1,26 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
ipld "github.com/ipfs/go-ipld-format"
)
// DagPutter is a general interface for a dag putter
type DagPutter interface {
DagPut(n ipld.Node) (string, error)
}
-86
View File
@@ -1,86 +0,0 @@
package mocks
import (
"context"
"errors"
"github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-exchange-interface"
)
// MockIPFSBlockService is a mock for testing the ipfs fetcher
type MockIPFSBlockService struct {
Blocks map[cid.Cid]blocks.Block
}
// GetBlock is used to retrieve a block from the mock BlockService
func (bs *MockIPFSBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
if bs.Blocks == nil {
return nil, errors.New("BlockService has not been initialized")
}
blk, ok := bs.Blocks[c]
if ok {
return blk, nil
}
return nil, nil
}
// GetBlocks is used to retrieve a set of blocks from the mock BlockService
func (bs *MockIPFSBlockService) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
if bs.Blocks == nil {
panic("BlockService has not been initialized")
}
blkChan := make(chan blocks.Block)
go func() {
for _, c := range cs {
blk, ok := bs.Blocks[c]
if ok {
blkChan <- blk
}
}
close(blkChan)
}()
return blkChan
}
// AddBlock adds a block to the mock BlockService
func (bs *MockIPFSBlockService) AddBlock(blk blocks.Block) error {
if bs.Blocks == nil {
bs.Blocks = make(map[cid.Cid]blocks.Block)
}
bs.Blocks[blk.Cid()] = blk
return nil
}
// AddBlocks adds a set of blocks to the mock BlockService
func (bs *MockIPFSBlockService) AddBlocks(blks []blocks.Block) error {
if bs.Blocks == nil {
bs.Blocks = make(map[cid.Cid]blocks.Block)
}
for _, block := range blks {
bs.Blocks[block.Cid()] = block
}
return nil
}
// Close is here to satisfy the interface
func (*MockIPFSBlockService) Close() error {
panic("implement me")
}
// Blockstore is here to satisfy the interface
func (*MockIPFSBlockService) Blockstore() blockstore.Blockstore {
panic("implement me")
}
// DeleteBlock is here to satisfy the interface
func (*MockIPFSBlockService) DeleteBlock(c cid.Cid) error {
panic("implement me")
}
// Exchange is here to satisfy the interface
func (*MockIPFSBlockService) Exchange() exchange.Interface {
panic("implement me")
}
-53
View File
@@ -1,53 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (
"errors"
node "github.com/ipfs/go-ipld-format"
"github.com/ethereum/go-ethereum/common"
)
// DagPutter is a mock for testing the ipfs publisher
type DagPutter struct {
PassedNode node.Node
ErrToReturn error
}
// DagPut returns the pre-loaded CIDs or error
func (dp *DagPutter) DagPut(n node.Node) (string, error) {
dp.PassedNode = n
return n.Cid().String(), dp.ErrToReturn
}
// MappedDagPutter is a mock for testing the ipfs publisher
type MappedDagPutter struct {
CIDsToReturn map[common.Hash]string
PassedNode node.Node
ErrToReturn error
}
// DagPut returns the pre-loaded CIDs or error
func (mdp *MappedDagPutter) DagPut(n node.Node) (string, error) {
if mdp.CIDsToReturn == nil {
return "", errors.New("mapped dag putter needs to be initialized with a map of cids to return")
}
hash := common.BytesToHash(n.RawData())
return mdp.CIDsToReturn[hash], nil
}