roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

cmd/swarm, swarm: added access control functionality (#17404)

Browse Source 
Co-authored-by: Janos Guljas <janos@resenje.org>
Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>
Co-authored-by: Balint Gabor <balint.g@gmail.com>
This commit is contained in:
Elad 2018-08-15 17:41:52 +02:00 committed by Balint Gabor
parent 040aa2bb10
commit e8752f4e9f
27 changed files with 1829 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

219
cmd/swarm/access.go Normal file
View File

@ -0,0 +1,219 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"crypto/rand"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
var salt = make([]byte, 32)
func init() {
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
}
func accessNewPass(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
accessKey []byte
err error
ref = args[0]
password = getPassPhrase("", 0, makePasswordList(ctx))
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
accessKey, ae, err = api.DoPasswordNew(ctx, password, salt)
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
if dryRun {
err = printManifests(m, nil)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
utils.Fatalf("uploading manifests")
err = uploadManifests(ctx, m, nil)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func accessNewPK(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
sessionKey []byte
err error
ref = args[0]
privateKey = getPrivKey(ctx)
granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name)
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt)
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
if dryRun {
err = printManifests(m, nil)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, nil)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func accessNewACT(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
actManifest *api.Manifest
accessKey []byte
err error
ref = args[0]
grantees = []string{}
actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
privateKey = getPrivKey(ctx)
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
bytes, err := ioutil.ReadFile(actFilename)
if err != nil {
utils.Fatalf("had an error reading the grantee public key list")
}
grantees = strings.Split(string(bytes), "\n")
accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees)
if err != nil {
utils.Fatalf("error generating ACT manifest: %v", err)
}
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
if err != nil {
utils.Fatalf("error generating root access manifest: %v", err)
}
if dryRun {
err = printManifests(m, actManifest)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, actManifest)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func printManifests(rootAccessManifest, actManifest *api.Manifest) error {
js, err := json.Marshal(rootAccessManifest)
if err != nil {
return err
}
fmt.Println(string(js))
if actManifest != nil {
js, err := json.Marshal(actManifest)
if err != nil {
return err
}
fmt.Println(string(js))
}
return nil
}
func uploadManifests(ctx *cli.Context, rootAccessManifest, actManifest *api.Manifest) error {
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := client.NewClient(bzzapi)
var (
key string
err error
)
if actManifest != nil {
key, err = client.UploadManifest(actManifest, false)
if err != nil {
return err
}
rootAccessManifest.Entries[0].Access.Act = key
}
key, err = client.UploadManifest(rootAccessManifest, false)
if err != nil {
return err
}
fmt.Println(key)
return nil
}
// makePasswordList reads password lines from the file specified by the global --password flag
// and also by the same subcommand --password flag.
// This function ia a fork of utils.MakePasswordList to lookup cli context for subcommand.
// Function ctx.SetGlobal is not setting the global flag value that can be accessed
// by ctx.GlobalString using the current version of cli package.
func makePasswordList(ctx *cli.Context) []string {
path := ctx.GlobalString(utils.PasswordFileFlag.Name)
if path == "" {
path = ctx.String(utils.PasswordFileFlag.Name)
if path == "" {
return nil
}
}
text, err := ioutil.ReadFile(path)
if err != nil {
utils.Fatalf("Failed to read password file: %v", err)
}
lines := strings.Split(string(text), "\n")
// Sanitise DOS line endings.
for i := range lines {
lines[i] = strings.TrimRight(lines[i], "\r")
}
return lines
}

581
cmd/swarm/access_test.go Normal file
View File

@ -0,0 +1,581 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"crypto/rand"
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
gorand "math/rand"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
)
// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
// is then fetched through 2nd node. since the tested code is not key-aware - we can just
// fetch from the 2nd node using HTTP BasicAuth
func TestAccessPassword(t *testing.T) {
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
proxyNode := cluster.Nodes[0]
// create a tmp file
tmp, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
// write data to file
data := "notsorandomdata"
dataFilename := filepath.Join(tmp, "data.txt")
err = ioutil.WriteFile(dataFilename, []byte(data), 0666)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{128}`
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
proxyNode.URL, //it doesn't matter through which node we upload content
"up",
"--encrypt",
dataFilename)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
password := "smth"
passwordFilename := filepath.Join(tmp, "password.txt")
err = ioutil.WriteFile(passwordFilename, []byte(password), 0666)
if err != nil {
t.Fatal(err)
}
up = runSwarm(t,
"access",
"new",
"pass",
"--dry-run",
"--password",
passwordFilename,
ref,
)
_, matches = up.ExpectRegexp(".+")
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
var m api.Manifest
err = json.Unmarshal([]byte(matches[0]), &m)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "pass" {
t.Errorf(`got access type %q, expected "pass"`, a.Type)
}
if len(a.Salt) < 32 {
t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.KdfParams == nil {
t.Fatal("manifest access kdf params is nil")
}
client := swarm.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusUnauthorized {
t.Fatal("should be a 401")
}
authHeader := response.Header.Get("WWW-Authenticate")
if authHeader == "" {
t.Fatal("should be something here")
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.SetBasicAuth("", password)
response, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
t.Errorf("expected status %v, got %v", http.StatusOK, response.StatusCode)
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt")
err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666)
if err != nil {
t.Fatal(err)
}
//download file with 'swarm down' with wrong password
up = runSwarm(t,
"--bzzapi",
proxyNode.URL,
"down",
"bzz:/"+hash,
tmp,
"--password",
wrongPasswordFilename)
_, matches = up.ExpectRegexp("unauthorized")
if len(matches) != 1 && matches[0] != "unauthorized" {
t.Fatal(`"unauthorized" not found in output"`)
}
up.ExpectExit()
}
// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
// the test will fail if the proxy's given private key is not granted on the ACT.
func TestAccessPK(t *testing.T) {
// Setup Swarm and upload a test file to it
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer tmp.Close()
defer os.Remove(tmp.Name())
// write data to file
data := "notsorandomdata"
_, err = io.WriteString(tmp, data)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{128}`
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
"--encrypt",
tmp.Name())
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
pk := cluster.Nodes[0].PrivateKey
granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
if err != nil {
t.Fatal(err)
}
passFile, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer passFile.Close()
defer os.Remove(passFile.Name())
_, err = io.WriteString(passFile, testPassphrase)
if err != nil {
t.Fatal(err)
}
_, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passFile.Name(),
"--datadir",
publisherDir,
"--bzzapi",
cluster.Nodes[0].URL,
"access",
"new",
"pk",
"--dry-run",
"--grant-key",
hex.EncodeToString(granteePubKey),
ref,
)
_, matches = up.ExpectRegexp(".+")
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
var m api.Manifest
err = json.Unmarshal([]byte(matches[0]), &m)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "pk" {
t.Errorf(`got access type %q, expected "pk"`, a.Type)
}
if len(a.Salt) < 32 {
t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.KdfParams != nil {
t.Fatal("manifest access kdf params should be nil")
}
client := swarm.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
}
// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control
// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through
// one of the nodes then disappears.
func TestAccessACT(t *testing.T) {
// Setup Swarm and upload a test file to it
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()
var uploadThroughNode = cluster.Nodes[0]
client := swarm.NewClient(uploadThroughNode.URL)
r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`)
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer tmp.Close()
defer os.Remove(tmp.Name())
// write data to file
data := "notsorandomdata"
_, err = io.WriteString(tmp, data)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{128}`
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
"--encrypt",
tmp.Name())
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
grantees := []string{}
for i, v := range cluster.Nodes {
if i == nodeToSkip {
continue
}
pk := v.PrivateKey
granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
grantees = append(grantees, hex.EncodeToString(granteePubKey))
}
granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv")
if err != nil {
t.Fatal(err)
}
_, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n"))
if err != nil {
t.Fatal(err)
}
defer granteesPubkeyListFile.Close()
defer os.Remove(granteesPubkeyListFile.Name())
publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
if err != nil {
t.Fatal(err)
}
passFile, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer passFile.Close()
defer os.Remove(passFile.Name())
_, err = io.WriteString(passFile, testPassphrase)
if err != nil {
t.Fatal(err)
}
_, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passFile.Name(),
"--datadir",
publisherDir,
"--bzzapi",
cluster.Nodes[0].URL,
"access",
"new",
"act",
"--grant-keys",
granteesPubkeyListFile.Name(),
ref,
)
_, matches = up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
hash := matches[0]
m, _, err := client.DownloadManifest(hash)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "act" {
t.Fatalf(`got access type %q, expected "act"`, a.Type)
}
if len(a.Salt) < 32 {
t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.KdfParams != nil {
t.Fatal("manifest access kdf params should be nil")
}
httpClient := &http.Client{}
// all nodes except the skipped node should be able to decrypt the content
for i, node := range cluster.Nodes {
log.Debug("trying to fetch from node", "node index", i)
url := node.URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
log.Debug("got response from node", "response code", response.StatusCode)
if i == nodeToSkip {
log.Debug("reached node to skip", "status code", response.StatusCode)
if response.StatusCode != http.StatusUnauthorized {
t.Fatalf("should be a 401")
}
continue
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
}
}
// TestKeypairSanity is a sanity test for the crypto scheme for ACT. it asserts the correct shared secret according to
// the specs at https://github.com/ethersphere/swarm-docs/blob/eb857afda906c6e7bb90d37f3f334ccce5eef230/act.md
func TestKeypairSanity(t *testing.T) {
salt := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
t.Fatalf("reading from crypto/rand failed: %v", err.Error())
}
sharedSecret := "a85586744a1ddd56a7ed9f33fa24f40dd745b3a941be296a0d60e329dbdb896d"
for i, v := range []struct {
publisherPriv string
granteePub string
}{
{
publisherPriv: "ec5541555f3bc6376788425e9d1a62f55a82901683fd7062c5eddcc373a73459",
granteePub: "0226f213613e843a413ad35b40f193910d26eb35f00154afcde9ded57479a6224a",
},
{
publisherPriv: "70c7a73011aa56584a0009ab874794ee7e5652fd0c6911cd02f8b6267dd82d2d",
granteePub: "02e6f8d5e28faaa899744972bb847b6eb805a160494690c9ee7197ae9f619181db",
},
} {
b, _ := hex.DecodeString(v.granteePub)
granteePub, _ := crypto.DecompressPubkey(b)
publisherPrivate, _ := crypto.HexToECDSA(v.publisherPriv)
ssKey, err := api.NewSessionKeyPK(publisherPrivate, granteePub, salt)
if err != nil {
t.Fatal(err)
}
hasher := sha3.NewKeccak256()
hasher.Write(salt)
shared, err := hex.DecodeString(sharedSecret)
if err != nil {
t.Fatal(err)
}
hasher.Write(shared)
sum := hasher.Sum(nil)
if !bytes.Equal(ssKey, sum) {
t.Fatalf("%d: got a session key mismatch", i)
}
}
}

1
cmd/swarm/config.go
View File

@ -78,6 +78,7 @@ const (
SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH"
SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY"
SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD"
GETH_ENV_DATADIR = "GETH_DATADIR"
)

40
cmd/swarm/download.go
View File

@ -68,18 +68,36 @@ func download(ctx *cli.Context) {
utils.Fatalf("could not parse uri argument: %v", err)
}
// assume behaviour according to --recursive switch
if isRecursive {
if err := client.DownloadDirectory(uri.Addr, uri.Path, dest); err != nil {
utils.Fatalf("encoutered an error while downloading directory: %v", err)
}
} else {
// we are downloading a file
log.Debug(fmt.Sprintf("downloading file/path from a manifest. hash: %s, path:%s", uri.Addr, uri.Path))
dl := func(credentials string) error {
// assume behaviour according to --recursive switch
if isRecursive {
if err := client.DownloadDirectory(uri.Addr, uri.Path, dest, credentials); err != nil {
if err == swarm.ErrUnauthorized {
return err
}
return fmt.Errorf("directory %s: %v", uri.Path, err)
}
} else {
// we are downloading a file
log.Debug("downloading file/path from a manifest", "uri.Addr", uri.Addr, "uri.Path", uri.Path)
err := client.DownloadFile(uri.Addr, uri.Path, dest)
if err != nil {
utils.Fatalf("could not download %s from given address: %s. error: %v", uri.Path, uri.Addr, err)
err := client.DownloadFile(uri.Addr, uri.Path, dest, credentials)
if err != nil {
if err == swarm.ErrUnauthorized {
return err
}
return fmt.Errorf("file %s from address: %s: %v", uri.Path, uri.Addr, err)
}
}
return nil
}
if passwords := makePasswordList(ctx); passwords != nil {
password := getPassPhrase(fmt.Sprintf("Downloading %s is restricted", uri), 0, passwords)
err = dl(password)
} else {
err = dl("")
}
if err != nil {
utils.Fatalf("download: %v", err)
}
}

2
cmd/swarm/list.go
View File

@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
list, err := client.List(manifest, prefix)
list, err := client.List(manifest, prefix, "")
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}

95
cmd/swarm/main.go
View File

@ -155,6 +155,14 @@ var (
Name: "defaultpath",
Usage: "path to file served for empty url path (none)",
}
SwarmAccessGrantKeyFlag = cli.StringFlag{
Name: "grant-key",
Usage: "grants a given public key access to an ACT",
}
SwarmAccessGrantKeysFlag = cli.StringFlag{
Name: "grant-keys",
Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
}
SwarmUpFromStdinFlag = cli.BoolFlag{
Name: "stdin",
Usage: "reads data to be uploaded from stdin",
@ -167,6 +175,15 @@ var (
Name: "encrypt",
Usage: "use encrypted upload",
}
SwarmAccessPasswordFlag = cli.StringFlag{
Name: "password",
Usage: "Password",
EnvVar: SWARM_ACCESS_PASSWORD,
}
SwarmDryRunFlag = cli.BoolFlag{
Name: "dry-run",
Usage: "dry-run",
}
CorsStringFlag = cli.StringFlag{
Name: "corsdomain",
Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
@ -252,6 +269,61 @@ func init() {
Flags: []cli.Flag{SwarmEncryptedFlag},
Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash",
},
{
CustomHelpTemplate: helpTemplate,
Name: "access",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root manifest",
Subcommands: []cli.Command{
{
CustomHelpTemplate: helpTemplate,
Name: "new",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
Subcommands: []cli.Command{
{
Action: accessNewPass,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
},
Name: "pass",
Usage: "encrypts a reference with a password and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewPK,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
SwarmAccessGrantKeyFlag,
},
Name: "pk",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewACT,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
SwarmAccessGrantKeysFlag,
SwarmDryRunFlag,
},
Name: "act",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
},
},
},
},
{
CustomHelpTemplate: helpTemplate,
Name: "resource",
@ -304,16 +376,13 @@ func init() {
Description: "Prints the swarm hash of file or directory",
},
{
Action: download,
Name: "down",
Flags: []cli.Flag{SwarmRecursiveFlag},
Usage: "downloads a swarm manifest or a file inside a manifest",
ArgsUsage: " <uri> [<dir>]",
Description: `
Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.
`,
Action: download,
Name: "down",
Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
Usage: "downloads a swarm manifest or a file inside a manifest",
ArgsUsage: " <uri> [<dir>]",
Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
},
{
Name: "manifest",
CustomHelpTemplate: helpTemplate,
@ -413,16 +482,14 @@ pv(1) tool to get a progress bar:
Name: "import",
Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
ArgsUsage: "<chunkdb> <file>",
Description: `
Import chunks from a tar archive into a local chunk database (use - to read from stdin).
Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
The import may be quite large, consider piping the input through the Unix
pv(1) tool to get a progress bar:
pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -
`,
pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
},
{
Action: dbClean,
@ -535,6 +602,7 @@ func version(ctx *cli.Context) error {
func bzzd(ctx *cli.Context) error {
//build a valid bzzapi.Config from all available sources:
//default config, file config, command line and env vars
bzzconfig, err := buildConfig(ctx)
if err != nil {
utils.Fatalf("unable to configure swarm: %v", err)
@ -557,6 +625,7 @@ func bzzd(ctx *cli.Context) error {
if err != nil {
utils.Fatalf("can't create node: %v", err)
}
//a few steps need to be done after the config phase is completed,
//due to overriding behavior
initSwarmNode(bzzconfig, stack, ctx)

25
cmd/swarm/run_test.go
View File

@ -18,10 +18,12 @@ package main
import (
"context"
"crypto/ecdsa"
"fmt"
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"runtime"
"sync"
@ -175,14 +177,15 @@ func (c *testCluster) Cleanup() {
}
type testNode struct {
Name string
Addr string
URL string
Enode string
Dir string
IpcPath string
Client *rpc.Client
Cmd *cmdtest.TestCmd
Name string
Addr string
URL string
Enode string
Dir string
IpcPath string
PrivateKey *ecdsa.PrivateKey
Client *rpc.Client
Cmd *cmdtest.TestCmd
}
const testPassphrase = "swarm-test-passphrase"
@ -289,7 +292,11 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
func newTestNode(t *testing.T, dir string) *testNode {
conf, account := getTestAccount(t, dir)
node := &testNode{Dir: dir}
ks := keystore.NewKeyStore(path.Join(dir, "keystore"), 1<<18, 1)
pk := decryptStoreAccount(ks, account.Address.Hex(), []string{testPassphrase})
node := &testNode{Dir: dir, PrivateKey: pk}
// assign ports
ports, err := getAvailableTCPPorts(2)

468
swarm/api/act.go Normal file
View File

@ -0,0 +1,468 @@
package api
import (
"context"
"crypto/ecdsa"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/ecies"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/sctx"
"github.com/ethereum/go-ethereum/swarm/storage"
"golang.org/x/crypto/scrypt"
cli "gopkg.in/urfave/cli.v1"
)
var (
ErrDecrypt = errors.New("cant decrypt - forbidden")
ErrUnknownAccessType = errors.New("unknown access type (or not implemented)")
ErrDecryptDomainForbidden = errors.New("decryption request domain forbidden - can only decrypt on localhost")
AllowedDecryptDomains = []string{
"localhost",
"127.0.0.1",
}
)
const EMPTY_CREDENTIALS = ""
type AccessEntry struct {
Type AccessType
Publisher string
Salt []byte
Act string
KdfParams *KdfParams
}
type DecryptFunc func(*ManifestEntry) error
func (a *AccessEntry) MarshalJSON() (out []byte, err error) {
return json.Marshal(struct {
Type AccessType `json:"type,omitempty"`
Publisher string `json:"publisher,omitempty"`
Salt string `json:"salt,omitempty"`
Act string `json:"act,omitempty"`
KdfParams *KdfParams `json:"kdf_params,omitempty"`
}{
Type: a.Type,
Publisher: a.Publisher,
Salt: hex.EncodeToString(a.Salt),
Act: a.Act,
KdfParams: a.KdfParams,
})
}
func (a *AccessEntry) UnmarshalJSON(value []byte) error {
v := struct {
Type AccessType `json:"type,omitempty"`
Publisher string `json:"publisher,omitempty"`
Salt string `json:"salt,omitempty"`
Act string `json:"act,omitempty"`
KdfParams *KdfParams `json:"kdf_params,omitempty"`
}{}
err := json.Unmarshal(value, &v)
if err != nil {
return err
}
a.Act = v.Act
a.KdfParams = v.KdfParams
a.Publisher = v.Publisher
a.Salt, err = hex.DecodeString(v.Salt)
if err != nil {
return err
}
if len(a.Salt) != 32 {
return errors.New("salt should be 32 bytes long")
}
a.Type = v.Type
return nil
}
type KdfParams struct {
N int `json:"n"`
P int `json:"p"`
R int `json:"r"`
}
type AccessType string
const AccessTypePass = AccessType("pass")
const AccessTypePK = AccessType("pk")
const AccessTypeACT = AccessType("act")
func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, error) {
if len(salt) != 32 {
return nil, fmt.Errorf("salt should be 32 bytes long")
}
return &AccessEntry{
Type: AccessTypePass,
Salt: salt,
KdfParams: kdfParams,
}, nil
}
func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) {
if len(publisher) != 66 {
return nil, fmt.Errorf("publisher should be 66 characters long, got %d", len(publisher))
}
if len(salt) != 32 {
return nil, fmt.Errorf("salt should be 32 bytes long")
}
return &AccessEntry{
Type: AccessTypePK,
Publisher: publisher,
Salt: salt,
}, nil
}
func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, error) {
if len(salt) != 32 {
return nil, fmt.Errorf("salt should be 32 bytes long")
}
if len(publisher) != 66 {
return nil, fmt.Errorf("publisher should be 66 characters long")
}
return &AccessEntry{
Type: AccessTypeACT,
Publisher: publisher,
Salt: salt,
Act: act,
}, nil
}
func NOOPDecrypt(*ManifestEntry) error {
return nil
}
var DefaultKdfParams = NewKdfParams(262144, 1, 8)
func NewKdfParams(n, p, r int) *KdfParams {
return &KdfParams{
N: n,
P: p,
R: r,
}
}
// NewSessionKeyPassword creates a session key based on a shared secret (password) and the given salt
// and kdf parameters in the access entry
func NewSessionKeyPassword(password string, accessEntry *AccessEntry) ([]byte, error) {
if accessEntry.Type != AccessTypePass {
return nil, errors.New("incorrect access entry type")
}
return scrypt.Key(
[]byte(password),
accessEntry.Salt,
accessEntry.KdfParams.N,
accessEntry.KdfParams.R,
accessEntry.KdfParams.P,
32,
)
}
// NewSessionKeyPK creates a new ACT Session Key using an ECDH shared secret for the given key pair and the given salt value
func NewSessionKeyPK(private *ecdsa.PrivateKey, public *ecdsa.PublicKey, salt []byte) ([]byte, error) {
granteePubEcies := ecies.ImportECDSAPublic(public)
privateKey := ecies.ImportECDSA(private)
bytes, err := privateKey.GenerateShared(granteePubEcies, 16, 16)
if err != nil {
return nil, err
}
bytes = append(salt, bytes...)
sessionKey := crypto.Keccak256(bytes)
return sessionKey, nil
}
func (a *API) NodeSessionKey(privateKey *ecdsa.PrivateKey, publicKey *ecdsa.PublicKey, salt []byte) ([]byte, error) {
return NewSessionKeyPK(privateKey, publicKey, salt)
}
func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.PrivateKey) DecryptFunc {
return func(m *ManifestEntry) error {
if m.Access == nil {
return nil
}
allowed := false
requestDomain := sctx.GetHost(ctx)
for _, v := range AllowedDecryptDomains {
if strings.Contains(requestDomain, v) {
allowed = true
}
}
if !allowed {
return ErrDecryptDomainForbidden
}
switch m.Access.Type {
case "pass":
if credentials != "" {
key, err := NewSessionKeyPassword(credentials, m.Access)
if err != nil {
return err
}
ref, err := hex.DecodeString(m.Hash)
if err != nil {
return err
}
enc := NewRefEncryption(len(ref) - 8)
decodedRef, err := enc.Decrypt(ref, key)
if err != nil {
return ErrDecrypt
}
m.Hash = hex.EncodeToString(decodedRef)
m.Access = nil
return nil
}
return ErrDecrypt
case "pk":
publisherBytes, err := hex.DecodeString(m.Access.Publisher)
if err != nil {
return ErrDecrypt
}
publisher, err := crypto.DecompressPubkey(publisherBytes)
if err != nil {
return ErrDecrypt
}
key, err := a.NodeSessionKey(pk, publisher, m.Access.Salt)
if err != nil {
return ErrDecrypt
}
ref, err := hex.DecodeString(m.Hash)
if err != nil {
return err
}
enc := NewRefEncryption(len(ref) - 8)
decodedRef, err := enc.Decrypt(ref, key)
if err != nil {
return ErrDecrypt
}
m.Hash = hex.EncodeToString(decodedRef)
m.Access = nil
return nil
case "act":
publisherBytes, err := hex.DecodeString(m.Access.Publisher)
if err != nil {
return ErrDecrypt
}
publisher, err := crypto.DecompressPubkey(publisherBytes)
if err != nil {
return ErrDecrypt
}
sessionKey, err := a.NodeSessionKey(pk, publisher, m.Access.Salt)
if err != nil {
return ErrDecrypt
}
hasher := sha3.NewKeccak256()
hasher.Write(append(sessionKey, 0))
lookupKey := hasher.Sum(nil)
hasher.Reset()
hasher.Write(append(sessionKey, 1))
accessKeyDecryptionKey := hasher.Sum(nil)
lk := hex.EncodeToString(lookupKey)
list, err := a.GetManifestList(ctx, NOOPDecrypt, storage.Address(common.Hex2Bytes(m.Access.Act)), lk)
found := ""
for _, v := range list.Entries {
if v.Path == lk {
found = v.Hash
}
}
if found == "" {
return ErrDecrypt
}
v, err := hex.DecodeString(found)
if err != nil {
return err
}
enc := NewRefEncryption(len(v) - 8)
decodedRef, err := enc.Decrypt(v, accessKeyDecryptionKey)
if err != nil {
return ErrDecrypt
}
ref, err := hex.DecodeString(m.Hash)
if err != nil {
return err
}
enc = NewRefEncryption(len(ref) - 8)
decodedMainRef, err := enc.Decrypt(ref, decodedRef)
if err != nil {
return ErrDecrypt
}
m.Hash = hex.EncodeToString(decodedMainRef)
m.Access = nil
return nil
}
return ErrUnknownAccessType
}
}
func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byte, ae *AccessEntry) (*Manifest, error) {
refBytes, err := hex.DecodeString(ref)
if err != nil {
return nil, err
}
// encrypt ref with accessKey
enc := NewRefEncryption(len(refBytes))
encrypted, err := enc.Encrypt(refBytes, accessKey)
if err != nil {
return nil, err
}
m := &Manifest{
Entries: []ManifestEntry{
{
Hash: hex.EncodeToString(encrypted),
ContentType: ManifestType,
ModTime: time.Now(),
Access: ae,
},
},
}
return m, nil
}
func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) {
if granteePublicKey == "" {
return nil, nil, errors.New("need a grantee Public Key")
}
b, err := hex.DecodeString(granteePublicKey)
if err != nil {
log.Error("error decoding grantee public key", "err", err)
return nil, nil, err
}
granteePub, err := crypto.DecompressPubkey(b)
if err != nil {
log.Error("error decompressing grantee public key", "err", err)
return nil, nil, err
}
sessionKey, err = NewSessionKeyPK(privateKey, granteePub, salt)
if err != nil {
log.Error("error getting session key", "err", err)
return nil, nil, err
}
ae, err = NewAccessEntryPK(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt)
if err != nil {
log.Error("error generating access entry", "err", err)
return nil, nil, err
}
return sessionKey, ae, nil
}
func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) {
if len(grantees) == 0 {
return nil, nil, nil, errors.New("did not get any grantee public keys")
}
publisherPub := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey))
grantees = append(grantees, publisherPub)
accessKey = make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
if _, err := io.ReadFull(rand.Reader, accessKey); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
lookupPathEncryptedAccessKeyMap := make(map[string]string)
i := 0
for _, v := range grantees {
i++
if v == "" {
return nil, nil, nil, errors.New("need a grantee Public Key")
}
b, err := hex.DecodeString(v)
if err != nil {
log.Error("error decoding grantee public key", "err", err)
return nil, nil, nil, err
}
granteePub, err := crypto.DecompressPubkey(b)
if err != nil {
log.Error("error decompressing grantee public key", "err", err)
return nil, nil, nil, err
}
sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt)
hasher := sha3.NewKeccak256()
hasher.Write(append(sessionKey, 0))
lookupKey := hasher.Sum(nil)
hasher.Reset()
hasher.Write(append(sessionKey, 1))
accessKeyEncryptionKey := hasher.Sum(nil)
enc := NewRefEncryption(len(accessKey))
encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey)
lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey)
}
m := &Manifest{
Entries: []ManifestEntry{},
}
for k, v := range lookupPathEncryptedAccessKeyMap {
m.Entries = append(m.Entries, ManifestEntry{
Path: k,
Hash: v,
ContentType: "text/plain",
})
}
ae, err = NewAccessEntryACT(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt, "")
if err != nil {
return nil, nil, nil, err
}
return accessKey, ae, m, nil
}
func DoPasswordNew(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) {
ae, err = NewAccessEntryPassword(salt, DefaultKdfParams)
if err != nil {
return nil, nil, err
}
sessionKey, err = NewSessionKeyPassword(password, ae)
if err != nil {
return nil, nil, err
}
return sessionKey, ae, nil
}

135
swarm/api/api.go
View File

@ -19,6 +19,9 @@ package api
import (
"archive/tar"
"context"
"crypto/ecdsa"
"encoding/hex"
"errors"
"fmt"
"io"
"math/big"
@ -43,6 +46,10 @@ import (
opentracing "github.com/opentracing/opentracing-go"
)
var (
ErrNotFound = errors.New("not found")
)
var (
apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil)
apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil)
@ -227,14 +234,18 @@ type API struct {
resource *mru.Handler
fileStore *storage.FileStore
dns Resolver
Decryptor func(context.Context, string) DecryptFunc
}
// NewAPI the api constructor initialises a new API instance.
func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler) (self *API) {
func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler, pk *ecdsa.PrivateKey) (self *API) {
self = &API{
fileStore: fileStore,
dns: dns,
resource: resourceHandler,
Decryptor: func(ctx context.Context, credentials string) DecryptFunc {
return self.doDecrypt(ctx, credentials, pk)
},
}
return
}
@ -260,8 +271,30 @@ func (a *API) Store(ctx context.Context, data io.Reader, size int64, toEncrypt b
// ErrResolve is returned when an URI cannot be resolved from ENS.
type ErrResolve error
// Resolve a name into a content-addressed hash
// where address could be an ENS name, or a content addressed hash
func (a *API) Resolve(ctx context.Context, address string) (storage.Address, error) {
// if DNS is not configured, return an error
if a.dns == nil {
if hashMatcher.MatchString(address) {
return common.Hex2Bytes(address), nil
}
apiResolveFail.Inc(1)
return nil, fmt.Errorf("no DNS to resolve name: %q", address)
}
// try and resolve the address
resolved, err := a.dns.Resolve(address)
if err != nil {
if hashMatcher.MatchString(address) {
return common.Hex2Bytes(address), nil
}
return nil, err
}
return resolved[:], nil
}
// Resolve resolves a URI to an Address using the MultiResolver.
func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) {
func (a *API) ResolveURI(ctx context.Context, uri *URI, credentials string) (storage.Address, error) {
apiResolveCount.Inc(1)
log.Trace("resolving", "uri", uri.Addr)
@ -280,28 +313,44 @@ func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) {
return key, nil
}
// if DNS is not configured, check if the address is a hash
if a.dns == nil {
key := uri.Address()
if key == nil {
apiResolveFail.Inc(1)
return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr)
}
return key, nil
}
// try and resolve the address
resolved, err := a.dns.Resolve(uri.Addr)
if err == nil {
return resolved[:], nil
}
key := uri.Address()
if key == nil {
apiResolveFail.Inc(1)
addr, err := a.Resolve(ctx, uri.Addr)
if err != nil {
return nil, err
}
return key, nil
if uri.Path == "" {
return addr, nil
}
walker, err := a.NewManifestWalker(ctx, addr, a.Decryptor(ctx, credentials), nil)
if err != nil {
return nil, err
}
var entry *ManifestEntry
walker.Walk(func(e *ManifestEntry) error {
// if the entry matches the path, set entry and stop
// the walk
if e.Path == uri.Path {
entry = e
// return an error to cancel the walk
return errors.New("found")
}
// ignore non-manifest files
if e.ContentType != ManifestType {
return nil
}
// if the manifest's path is a prefix of the
// requested path, recurse into it by returning
// nil and continuing the walk
if strings.HasPrefix(uri.Path, e.Path) {
return nil
}
return ErrSkipManifest
})
if entry == nil {
return nil, errors.New("not found")
}
addr = storage.Address(common.Hex2Bytes(entry.Hash))
return addr, nil
}
// Put provides singleton manifest creation on top of FileStore store
@ -332,10 +381,10 @@ func (a *API) Put(ctx context.Context, content string, contentType string, toEnc
// Get uses iterative manifest retrieval and prefix matching
// to resolve basePath to content using FileStore retrieve
// it returns a section reader, mimeType, status, the key of the actual content and an error
func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) {
func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) {
log.Debug("api.get", "key", manifestAddr, "path", path)
apiGetCount.Inc(1)
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil)
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
@ -347,6 +396,16 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
if entry != nil {
log.Debug("trie got entry", "key", manifestAddr, "path", path, "entry.Hash", entry.Hash)
if entry.ContentType == ManifestType {
log.Debug("entry is manifest", "key", manifestAddr, "new key", entry.Hash)
adr, err := hex.DecodeString(entry.Hash)
if err != nil {
return nil, "", 0, nil, err
}
return a.Get(ctx, decrypt, adr, entry.Path)
}
// we need to do some extra work if this is a mutable resource manifest
if entry.ContentType == ResourceContentType {
@ -398,7 +457,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
log.Trace("resource is multihash", "key", manifestAddr)
// get the manifest the multihash digest points to
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil)
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
@ -451,7 +510,7 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add
apiDeleteFail.Inc(1)
return nil, err
}
key, err := a.Resolve(ctx, uri)
key, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
if err != nil {
return nil, err
@ -470,13 +529,13 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add
// GetDirectoryTar fetches a requested directory as a tarstream
// it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser
func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, error) {
func (a *API) GetDirectoryTar(ctx context.Context, decrypt DecryptFunc, uri *URI) (io.ReadCloser, error) {
apiGetTarCount.Inc(1)
addr, err := a.Resolve(ctx, uri)
addr, err := a.Resolve(ctx, uri.Addr)
if err != nil {
return nil, err
}
walker, err := a.NewManifestWalker(ctx, addr, nil)
walker, err := a.NewManifestWalker(ctx, addr, decrypt, nil)
if err != nil {
apiGetTarFail.Inc(1)
return nil, err
@ -542,9 +601,9 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err
// GetManifestList lists the manifest entries for the specified address and prefix
// and returns it as a ManifestList
func (a *API) GetManifestList(ctx context.Context, addr storage.Address, prefix string) (list ManifestList, err error) {
func (a *API) GetManifestList(ctx context.Context, decryptor DecryptFunc, addr storage.Address, prefix string) (list ManifestList, err error) {
apiManifestListCount.Inc(1)
walker, err := a.NewManifestWalker(ctx, addr, nil)
walker, err := a.NewManifestWalker(ctx, addr, decryptor, nil)
if err != nil {
apiManifestListFail.Inc(1)
return ManifestList{}, err
@ -631,7 +690,7 @@ func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update f
func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) {
apiModifyCount.Inc(1)
quitC := make(chan bool)
trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
if err != nil {
apiModifyFail.Inc(1)
return nil, err
@ -663,7 +722,7 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content []
apiAddFileFail.Inc(1)
return nil, "", err
}
mkey, err := a.Resolve(ctx, uri)
mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
@ -770,7 +829,7 @@ func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname s
apiRmFileFail.Inc(1)
return "", err
}
mkey, err := a.Resolve(ctx, uri)
mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
@ -837,7 +896,7 @@ func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existin
apiAppendFileFail.Inc(1)
return nil, "", err
}
mkey, err := a.Resolve(ctx, uri)
mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
@ -891,13 +950,13 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver
if err != nil {
return nil, nil, err
}
addr, err = a.Resolve(ctx, uri)
addr, err = a.Resolve(ctx, uri.Addr)
if err != nil {
return nil, nil, err
}
quitC := make(chan bool)
rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC)
rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
if err != nil {
return nil, nil, fmt.Errorf("can't load manifest %v: %v", addr.String(), err)
}
@ -955,7 +1014,7 @@ func (a *API) ResourceHashSize() int {
// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk.
func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) {
trie, err := loadManifest(ctx, a.fileStore, addr, nil)
trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt)
if err != nil {
return nil, fmt.Errorf("cannot load resource manifest: %v", err)
}

68
swarm/api/api_test.go
View File

@ -19,6 +19,7 @@ package api
import (
"context"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
@ -28,10 +29,17 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/sctx"
"github.com/ethereum/go-ethereum/swarm/storage"
)
func init() {
loglevel := flag.Int("loglevel", 2, "loglevel")
flag.Parse()
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
}
func testAPI(t *testing.T, f func(*API, bool)) {
datadir, err := ioutil.TempDir("", "bzz-test")
if err != nil {
@ -42,7 +50,7 @@ func testAPI(t *testing.T, f func(*API, bool)) {
if err != nil {
return
}
api := NewAPI(fileStore, nil, nil)
api := NewAPI(fileStore, nil, nil, nil)
f(api, false)
f(api, true)
}
@ -85,7 +93,7 @@ func expResponse(content string, mimeType string, status int) *Response {
func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse {
addr := storage.Address(common.Hex2Bytes(bzzhash))
reader, mimeType, status, _, err := api.Get(context.TODO(), addr, path)
reader, mimeType, status, _, err := api.Get(context.TODO(), NOOPDecrypt, addr, path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -229,7 +237,7 @@ func TestAPIResolve(t *testing.T) {
if x.immutable {
uri.Scheme = "bzz-immutable"
}
res, err := api.Resolve(context.TODO(), uri)
res, err := api.ResolveURI(context.TODO(), uri, "")
if err == nil {
if x.expectErr != nil {
t.Fatalf("expected error %q, got result %q", x.expectErr, res)
@ -373,3 +381,55 @@ func TestMultiResolver(t *testing.T) {
})
}
}
func TestDecryptOriginForbidden(t *testing.T) {
ctx := context.TODO()
ctx = sctx.SetHost(ctx, "swarm-gateways.net")
me := &ManifestEntry{
Access: &AccessEntry{Type: AccessTypePass},
}
api := NewAPI(nil, nil, nil, nil)
f := api.Decryptor(ctx, "")
err := f(me)
if err != ErrDecryptDomainForbidden {
t.Fatalf("should fail with ErrDecryptDomainForbidden, got %v", err)
}
}
func TestDecryptOrigin(t *testing.T) {
for _, v := range []struct {
host string
expectError error
}{
{
host: "localhost",
expectError: ErrDecrypt,
},
{
host: "127.0.0.1",
expectError: ErrDecrypt,
},
{
host: "swarm-gateways.net",
expectError: ErrDecryptDomainForbidden,
},
} {
ctx := context.TODO()
ctx = sctx.SetHost(ctx, v.host)
me := &ManifestEntry{
Access: &AccessEntry{Type: AccessTypePass},
}
api := NewAPI(nil, nil, nil, nil)
f := api.Decryptor(ctx, "")
err := f(me)
if err != v.expectError {
t.Fatalf("should fail with %v, got %v", v.expectError, err)
}
}
}

48
swarm/api/client/client.go
View File

@ -43,6 +43,10 @@ var (
DefaultClient = NewClient(DefaultGateway)
)
var (
ErrUnauthorized = errors.New("unauthorized")
)
func NewClient(gateway string) *Client {
return &Client{
Gateway: gateway,
@ -188,7 +192,7 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo
// DownloadDirectory downloads the files contained in a swarm manifest under
// the given path into a local directory (existing files will be overwritten)
func (c *Client) DownloadDirectory(hash, path, destDir string) error {
func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error {
stat, err := os.Stat(destDir)
if err != nil {
return err
@ -201,13 +205,20 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error {
if err != nil {
return err
}
if credentials != "" {
req.SetBasicAuth("", credentials)
}
req.Header.Set("Accept", "application/x-tar")
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
switch res.StatusCode {
case http.StatusOK:
case http.StatusUnauthorized:
return ErrUnauthorized
default:
return fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
tr := tar.NewReader(res.Body)
@ -248,7 +259,7 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error {
// DownloadFile downloads a single file into the destination directory
// if the manifest entry does not specify a file name - it will fallback
// to the hash of the file as a filename
func (c *Client) DownloadFile(hash, path, dest string) error {
func (c *Client) DownloadFile(hash, path, dest, credentials string) error {
hasDestinationFilename := false
if stat, err := os.Stat(dest); err == nil {
hasDestinationFilename = !stat.IsDir()
@ -261,9 +272,9 @@ func (c *Client) DownloadFile(hash, path, dest string) error {
}
}
manifestList, err := c.List(hash, path)
manifestList, err := c.List(hash, path, credentials)
if err != nil {
return fmt.Errorf("could not list manifest: %v", err)
return err
}
switch len(manifestList.Entries) {
@ -280,13 +291,19 @@ func (c *Client) DownloadFile(hash, path, dest string) error {
if err != nil {
return err
}
if credentials != "" {
req.SetBasicAuth("", credentials)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
switch res.StatusCode {
case http.StatusOK:
case http.StatusUnauthorized:
return ErrUnauthorized
default:
return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode)
}
filename := ""
@ -367,13 +384,24 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) {
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
//
// where entries ending with "/" are common prefixes.
func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) {
req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil)
if err != nil {
return nil, err
}
if credentials != "" {
req.SetBasicAuth("", credentials)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
switch res.StatusCode {
case http.StatusOK:
case http.StatusUnauthorized:
return nil, ErrUnauthorized
default:
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
var list api.ManifestList

4
swarm/api/client/client_test.go
View File

@ -228,7 +228,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
if err := client.DownloadDirectory(hash, "", tmp); err != nil {
if err := client.DownloadDirectory(hash, "", tmp, ""); err != nil {
t.Fatal(err)
}
for _, file := range testDirFiles {
@ -265,7 +265,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) {
}
ls := func(prefix string) []string {
list, err := client.List(hash, prefix)
list, err := client.List(hash, prefix, "")
if err != nil {
t.Fatal(err)
}

76
swarm/api/encrypt.go Normal file
View File

@ -0,0 +1,76 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"encoding/binary"
"errors"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/swarm/storage/encryption"
)
type RefEncryption struct {
spanEncryption encryption.Encryption
dataEncryption encryption.Encryption
span []byte
}
func NewRefEncryption(refSize int) *RefEncryption {
span := make([]byte, 8)
binary.LittleEndian.PutUint64(span, uint64(refSize))
return &RefEncryption{
spanEncryption: encryption.New(0, uint32(refSize/32), sha3.NewKeccak256),
dataEncryption: encryption.New(refSize, 0, sha3.NewKeccak256),
span: span,
}
}
func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) {
encryptedSpan, err := re.spanEncryption.Encrypt(re.span, key)
if err != nil {
return nil, err
}
encryptedData, err := re.dataEncryption.Encrypt(ref, key)
if err != nil {
return nil, err
}
encryptedRef := make([]byte, len(ref)+8)
copy(encryptedRef[:8], encryptedSpan)
copy(encryptedRef[8:], encryptedData)
return encryptedRef, nil
}
func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) {
decryptedSpan, err := re.spanEncryption.Decrypt(ref[:8], key)
if err != nil {
return nil, err
}
size := binary.LittleEndian.Uint64(decryptedSpan)
if size != uint64(len(ref)-8) {
return nil, errors.New("invalid span in encrypted reference")
}
decryptedRef, err := re.dataEncryption.Decrypt(ref[8:], key)
if err != nil {
return nil, err
}
return decryptedRef, nil
}

4
swarm/api/filesystem.go
View File

@ -191,7 +191,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error {
if err != nil {
return err
}
addr, err := fs.api.Resolve(context.TODO(), uri)
addr, err := fs.api.Resolve(context.TODO(), uri.Addr)
if err != nil {
return err
}
@ -202,7 +202,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error {
}
quitC := make(chan bool)
trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC)
trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC, NOOPDecrypt)
if err != nil {
log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err))
return err

4
swarm/api/filesystem_test.go
View File

@ -64,7 +64,7 @@ func TestApiDirUpload0(t *testing.T) {
checkResponse(t, resp, exp)
addr := storage.Address(common.Hex2Bytes(bzzhash))
_, _, _, _, err = api.Get(context.TODO(), addr, "")
_, _, _, _, err = api.Get(context.TODO(), NOOPDecrypt, addr, "")
if err == nil {
t.Fatalf("expected error: %v", err)
}
@ -143,7 +143,7 @@ func TestApiDirUploadModify(t *testing.T) {
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
_, _, _, _, err = api.Get(context.TODO(), addr, "")
_, _, _, _, err = api.Get(context.TODO(), nil, addr, "")
if err == nil {
t.Errorf("expected error: %v", err)
}

12
swarm/api/http/middleware.go
View File

@ -9,6 +9,7 @@ import (
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/sctx"
"github.com/ethereum/go-ethereum/swarm/spancontext"
"github.com/pborman/uuid"
)
@ -35,6 +36,15 @@ func SetRequestID(h http.Handler) http.Handler {
})
}
func SetRequestHost(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(sctx.SetHost(r.Context(), r.Host))
log.Info("setting request host", "ruid", GetRUID(r.Context()), "host", sctx.GetHost(r.Context()))
h.ServeHTTP(w, r)
})
}
func ParseURI(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
@ -87,7 +97,7 @@ func RecoverPanic(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header)
log.Error("panic recovery!", "stack trace", string(debug.Stack()), "url", r.URL.String(), "headers", r.Header)
}
}()
h.ServeHTTP(w, r)

2
swarm/api/http/response.go
View File

@ -79,7 +79,7 @@ func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s
}
func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code)
RespondTemplate(w, r, "error", msg, code)
}

111
swarm/api/http/server.go
View File

@ -23,7 +23,6 @@ import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -97,6 +96,7 @@ func NewServer(api *api.API, corsString string) *Server {
defaultMiddlewares := []Adapter{
RecoverPanic,
SetRequestID,
SetRequestHost,
InitLoggingResponseWriter,
ParseURI,
InstrumentOpenTracing,
@ -169,6 +169,7 @@ func NewServer(api *api.API, corsString string) *Server {
}
func (s *Server) ListenAndServe(addr string) error {
s.listenAddr = addr
return http.ListenAndServe(addr, s)
}
@ -178,16 +179,24 @@ func (s *Server) ListenAndServe(addr string) error {
// https://github.com/atom/electron/blob/master/docs/api/protocol.md
type Server struct {
http.Handler
api *api.API
api *api.API
listenAddr string
}
func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()))
log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI)
if r.Header.Get("Accept") == "application/x-tar" {
uri := GetURI(r.Context())
reader, err := s.api.GetDirectoryTar(r.Context(), uri)
_, credentials, _ := r.BasicAuth()
reader, err := s.api.GetDirectoryTar(r.Context(), s.api.Decryptor(r.Context(), credentials), uri)
if err != nil {
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String()))
RespondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
return
}
defer reader.Close()
@ -287,7 +296,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
var addr storage.Address
if uri.Addr != "" && uri.Addr != "encrypt" {
addr, err = s.api.Resolve(r.Context(), uri)
addr, err = s.api.Resolve(r.Context(), uri.Addr)
if err != nil {
postFilesFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError)
@ -563,7 +572,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) {
// resolve the content key.
manifestAddr := uri.Address()
if manifestAddr == nil {
manifestAddr, err = s.api.Resolve(r.Context(), uri)
manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr)
if err != nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
@ -682,62 +691,21 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
uri := GetURI(r.Context())
log.Debug("handle.get", "ruid", ruid, "uri", uri)
getCount.Inc(1)
_, pass, _ := r.BasicAuth()
var err error
addr := uri.Address()
if addr == nil {
addr, err = s.api.Resolve(r.Context(), uri)
if err != nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
} else {
w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
addr, err := s.api.ResolveURI(r.Context(), uri, pass)
if err != nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
log.Debug("handle.get: resolved", "ruid", ruid, "key", addr)
// if path is set, interpret <key> as a manifest and return the
// raw entry at the given path
if uri.Path != "" {
walker, err := s.api.NewManifestWalker(r.Context(), addr, nil)
if err != nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
return
}
var entry *api.ManifestEntry
walker.Walk(func(e *api.ManifestEntry) error {
// if the entry matches the path, set entry and stop
// the walk
if e.Path == uri.Path {
entry = e
// return an error to cancel the walk
return errors.New("found")
}
// ignore non-manifest files
if e.ContentType != api.ManifestType {
return nil
}
// if the manifest's path is a prefix of the
// requested path, recurse into it by returning
// nil and continuing the walk
if strings.HasPrefix(uri.Path, e.Path) {
return nil
}
return api.ErrSkipManifest
})
if entry == nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
return
}
addr = storage.Address(common.Hex2Bytes(entry.Hash))
}
etag := common.Bytes2Hex(addr)
noneMatchEtag := r.Header.Get("If-None-Match")
w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key.
@ -781,6 +749,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
_, credentials, _ := r.BasicAuth()
log.Debug("handle.get.list", "ruid", ruid, "uri", uri)
getListCount.Inc(1)
@ -790,7 +759,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
return
}
addr, err := s.api.Resolve(r.Context(), uri)
addr, err := s.api.Resolve(r.Context(), uri.Addr)
if err != nil {
getListFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
@ -798,9 +767,14 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
}
log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr)
list, err := s.api.GetManifestList(r.Context(), addr, uri.Path)
list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), addr, uri.Path)
if err != nil {
getListFail.Inc(1)
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String()))
RespondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
@ -833,7 +807,8 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.get.file", "ruid", ruid)
_, credentials, _ := r.BasicAuth()
log.Debug("handle.get.file", "ruid", ruid, "uri", r.RequestURI)
getFileCount.Inc(1)
// ensure the root path has a trailing slash so that relative URLs work
@ -845,7 +820,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
manifestAddr := uri.Address()
if manifestAddr == nil {
manifestAddr, err = s.api.Resolve(r.Context(), uri)
manifestAddr, err = s.api.ResolveURI(r.Context(), uri, credentials)
if err != nil {
getFileFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
@ -856,7 +831,8 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
}
log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr)
reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path)
reader, contentType, status, contentKey, err := s.api.Get(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
etag := common.Bytes2Hex(contentKey)
noneMatchEtag := r.Header.Get("If-None-Match")
@ -869,6 +845,12 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
}
if err != nil {
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
RespondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
switch status {
case http.StatusNotFound:
getFileNotFound.Inc(1)
@ -883,9 +865,14 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
//the request results in ambiguous files
//e.g. /read with readme.md and readinglist.txt available in manifest
if status == http.StatusMultipleChoices {
list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path)
list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
if err != nil {
getFileFail.Inc(1)
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
RespondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
@ -951,3 +938,7 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
func isDecryptError(err error) bool {
return strings.Contains(err.Error(), api.ErrDecrypt.Error())
}

69
swarm/api/manifest.go
View File

@ -46,13 +46,14 @@ type Manifest struct {
// ManifestEntry represents an entry in a swarm manifest
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
Access *AccessEntry `json:"access,omitempty"`
}
// ManifestList represents the result of listing files in a manifest
@ -98,7 +99,7 @@ type ManifestWriter struct {
}
func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) {
trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
}
@ -141,8 +142,8 @@ type ManifestWalker struct {
quitC chan bool
}
func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWalker, error) {
trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, decrypt DecryptFunc, quitC chan bool) (*ManifestWalker, error) {
trie, err := loadManifest(ctx, a.fileStore, addr, quitC, decrypt)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
}
@ -194,6 +195,7 @@ type manifestTrie struct {
entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry
ref storage.Address // if ref != nil, it is stored
encrypted bool
decrypt DecryptFunc
}
func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
@ -209,15 +211,15 @@ type manifestTrieEntry struct {
subtrie *manifestTrie
}
func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
log.Trace("manifest lookup", "key", hash)
// retrieve manifest via FileStore
manifestReader, isEncrypted := fileStore.Retrieve(ctx, hash)
log.Trace("reader retrieved", "key", hash)
return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC)
return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC, decrypt)
}
func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
// TODO check size for oversized manifests
size, err := mr.Size(mr.Context(), quitC)
@ -258,26 +260,41 @@ func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore
trie = &manifestTrie{
fileStore: fileStore,
encrypted: isEncrypted,
decrypt: decrypt,
}
for _, entry := range man.Entries {
trie.addEntry(entry, quitC)
err = trie.addEntry(entry, quitC)
if err != nil {
return
}
}
return
}
func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) error {
mt.ref = nil // trie modified, hash needs to be re-calculated on demand
if entry.ManifestEntry.Access != nil {
if mt.decrypt == nil {
return errors.New("dont have decryptor")
}
err := mt.decrypt(&entry.ManifestEntry)
if err != nil {
return err
}
}
if len(entry.Path) == 0 {
mt.entries[256] = entry
return
return nil
}
b := entry.Path[0]
oldentry := mt.entries[b]
if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) {
mt.entries[b] = entry
return
return nil
}
cpl := 0
@ -287,12 +304,12 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
if mt.loadSubTrie(oldentry, quitC) != nil {
return
return nil
}
entry.Path = entry.Path[cpl:]
oldentry.subtrie.addEntry(entry, quitC)
oldentry.Hash = ""
return
return nil
}
commonPrefix := entry.Path[:cpl]
@ -310,6 +327,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
Path: commonPrefix,
ContentType: ManifestType,
}, subtrie)
return nil
}
func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
@ -398,9 +416,20 @@ func (mt *manifestTrie) recalcAndStore() error {
}
func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
if entry.ManifestEntry.Access != nil {
if mt.decrypt == nil {
return errors.New("dont have decryptor")
}
err := mt.decrypt(&entry.ManifestEntry)
if err != nil {
return err
}
}
if entry.subtrie == nil {
hash := common.Hex2Bytes(entry.Hash)
entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC)
entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC, mt.decrypt)
entry.Hash = "" // might not match, should be recalculated
}
return

8
swarm/api/manifest_test.go
View File

@ -44,7 +44,7 @@ func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...stri
quitC := make(chan bool)
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC)
trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC, NOOPDecrypt)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
@ -101,7 +101,7 @@ func TestExactMatch(t *testing.T) {
mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map")
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(mf, ref, fileStore, false, quitC)
trie, err := readManifest(mf, ref, fileStore, false, quitC, nil)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
@ -134,7 +134,7 @@ func TestAddFileWithManifestPath(t *testing.T) {
}
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(reader, ref, fileStore, false, nil)
trie, err := readManifest(reader, ref, fileStore, false, nil, NOOPDecrypt)
if err != nil {
t.Fatal(err)
}
@ -161,7 +161,7 @@ func TestReadManifestOverSizeLimit(t *testing.T) {
reader := &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
}
_, err := readManifest(reader, storage.Address{}, nil, false, nil)
_, err := readManifest(reader, storage.Address{}, nil, false, nil, NOOPDecrypt)
if err == nil {
t.Fatal("got no error from readManifest")
}

6
swarm/api/storage.go
View File

@ -63,11 +63,11 @@ func (s *Storage) Get(ctx context.Context, bzzpath string) (*Response, error) {
if err != nil {
return nil, err
}
addr, err := s.api.Resolve(ctx, uri)
addr, err := s.api.Resolve(ctx, uri.Addr)
if err != nil {
return nil, err
}
reader, mimeType, status, _, err := s.api.Get(ctx, addr, uri.Path)
reader, mimeType, status, _, err := s.api.Get(ctx, nil, addr, uri.Path)
if err != nil {
return nil, err
}
@ -93,7 +93,7 @@ func (s *Storage) Modify(ctx context.Context, rootHash, path, contentHash, conte
if err != nil {
return "", err
}
addr, err := s.api.Resolve(ctx, uri)
addr, err := s.api.Resolve(ctx, uri.Addr)
if err != nil {
return "", err
}

13
swarm/api/uri.go
View File

@ -53,6 +53,19 @@ type URI struct {
Path string
}
func (u *URI) MarshalJSON() (out []byte, err error) {
return []byte(`"` + u.String() + `"`), nil
}
func (u *URI) UnmarshalJSON(value []byte) error {
uri, err := Parse(string(value))
if err != nil {
return err
}
*u = *uri
return nil
}
// Parse parses rawuri into a URI struct, where rawuri is expected to have one
// of the following formats:
//

2
swarm/fuse/swarmfs_test.go
View File

@ -1650,7 +1650,7 @@ func TestFUSE(t *testing.T) {
if err != nil {
t.Fatal(err)
}
ta := &testAPI{api: api.NewAPI(fileStore, nil, nil)}
ta := &testAPI{api: api.NewAPI(fileStore, nil, nil, nil)}
//run a short suite of tests
//approx time: 28s

2
swarm/network_test.go
View File

@ -445,7 +445,7 @@ func retrieve(
log.Debug("api get: check file", "node", id.String(), "key", f.addr.String(), "total files found", atomic.LoadUint64(totalFoundCount))
r, _, _, _, err := swarm.api.Get(context.TODO(), f.addr, "/")
r, _, _, _, err := swarm.api.Get(context.TODO(), api.NOOPDecrypt, f.addr, "/")
if err != nil {
errc <- fmt.Errorf("api get: node %s, key %s, kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err)
return

15
swarm/sctx/sctx.go
View File

@ -1,7 +1,22 @@
package sctx
import "context"
type ContextKey int
const (
HTTPRequestIDKey ContextKey = iota
requestHostKey
)
func SetHost(ctx context.Context, domain string) context.Context {
return context.WithValue(ctx, requestHostKey, domain)
}
func GetHost(ctx context.Context) string {
v, ok := ctx.Value(requestHostKey).(string)
if ok {
return v
}
return ""
}

4
swarm/swarm.go
View File

@ -85,14 +85,12 @@ type Swarm struct {
type SwarmAPI struct {
Api *api.API
Backend chequebook.Backend
PrvKey *ecdsa.PrivateKey
}
func (self *Swarm) API() *SwarmAPI {
return &SwarmAPI{
Api: self.api,
Backend: self.backend,
PrvKey: self.privateKey,
}
}
@ -217,7 +215,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
pss.SetHandshakeController(self.ps, pss.NewHandshakeParams())
}
self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler)
self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey)
// Manifests for Smart Hosting
log.Debug(fmt.Sprintf("-> Web3 virtual server API"))

2
swarm/testutil/http.go
View File

@ -77,7 +77,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *Tes
t.Fatal(err)
}
a := api.NewAPI(fileStore, nil, rh.Handler)
a := api.NewAPI(fileStore, nil, rh.Handler, nil)
srv := httptest.NewServer(serverFunc(a))
return &TestSwarmServer{
Server: srv,