From 752f9f4d7a3dfcb12da1b0611f64e54540092b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nemanja=20Zbilji=C4=87?= Date: Thu, 20 Aug 2020 17:59:04 +0200 Subject: [PATCH 1/3] Add support for 'mantaray' manifest type --- go.mod | 4 +- go.sum | 6 +- pkg/manifest/manifest.go | 4 + pkg/manifest/mantaray.go | 176 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 pkg/manifest/mantaray.go diff --git a/go.mod b/go.mod index c7ae30bfa9c..b8008202bf2 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/coreos/go-semver v0.3.0 github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/ethersphere/bmt v0.1.2 - github.com/ethersphere/manifest v0.1.0 + github.com/ethersphere/manifest v0.2.0 github.com/gogo/protobuf v1.3.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect @@ -51,7 +51,7 @@ require ( gitlab.com/nolash/go-mockbytes v0.0.7 go.opencensus.io v0.22.4 // indirect go.uber.org/zap v1.15.0 // indirect - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/mod v0.3.0 // indirect golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 diff --git a/go.sum b/go.sum index 0aba5b172dc..6f7d11a5cde 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethersphere/bmt v0.1.2 h1:FEuvQY9xuK+rDp3VwDVyde8T396Matv/u9PdtKa2r9Q= github.com/ethersphere/bmt v0.1.2/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws= -github.com/ethersphere/manifest v0.1.0 h1:uVlFzAZk5SqyzjHzDgF3rNuDA4CdbJQ8fVHS4pN0iHY= -github.com/ethersphere/manifest v0.1.0/go.mod h1:eV7hOz2c5R1ol+SpBYdS5EUG6ubh11CQe6lFAn5q+q4= +github.com/ethersphere/manifest v0.2.0 h1:HD2ufiIaw/5Vgrl4XyeGduDJ5tn50wIhqMQoWdT2GME= +github.com/ethersphere/manifest v0.2.0/go.mod h1:ygAx0KLhXYmKqsjUab95RCbXf8UcO7yMDjyfP0lY76Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= @@ -887,6 +887,8 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index fff5782ec04..a441e061e57 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -60,6 +60,8 @@ func NewManifest( switch manifestType { case ManifestSimpleContentType: return NewSimpleManifest(encrypted, storer) + case ManifestMantarayContentType: + return NewMantarayManifest(encrypted, storer) default: return nil, ErrInvalidManifestType } @@ -76,6 +78,8 @@ func NewManifestReference( switch manifestType { case ManifestSimpleContentType: return NewSimpleManifestReference(ctx, reference, encrypted, storer) + case ManifestMantarayContentType: + return NewMantarayManifestReference(ctx, reference, encrypted, storer) default: return nil, ErrInvalidManifestType } diff --git a/pkg/manifest/mantaray.go b/pkg/manifest/mantaray.go new file mode 100644 index 00000000000..110a99d7328 --- /dev/null +++ b/pkg/manifest/mantaray.go @@ -0,0 +1,176 @@ +// Copyright 2020 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package manifest + +import ( + "bytes" + "context" + "errors" + "fmt" + + "github.com/ethersphere/bee/pkg/file" + "github.com/ethersphere/bee/pkg/file/joiner" + "github.com/ethersphere/bee/pkg/file/splitter" + "github.com/ethersphere/bee/pkg/storage" + "github.com/ethersphere/bee/pkg/swarm" + "github.com/ethersphere/manifest/mantaray" +) + +const ( + // ManifestMantarayContentType represents content type used for noting that + // specific file should be processed as mantaray manifest. + ManifestMantarayContentType = "application/bzz-manifest-mantaray+octet-stream" +) + +type mantarayManifest struct { + trie *mantaray.Node + + encrypted bool + storer storage.Storer + + loader mantaray.LoadSaver +} + +// NewMantarayManifest creates a new mantaray-based manifest. +func NewMantarayManifest( + encrypted bool, + storer storage.Storer, +) (Interface, error) { + return &mantarayManifest{ + trie: mantaray.New(), + encrypted: encrypted, + storer: storer, + }, nil +} + +// NewMantarayManifestReference loads existing mantaray-based manifest. +func NewMantarayManifestReference( + ctx context.Context, + reference swarm.Address, + encrypted bool, + storer storage.Storer, +) (Interface, error) { + return &mantarayManifest{ + trie: mantaray.NewNodeRef(reference.Bytes()), + encrypted: encrypted, + storer: storer, + loader: newMantarayLoader(ctx, encrypted, storer), + }, nil +} + +func (m *mantarayManifest) Type() string { + return ManifestMantarayContentType +} + +func (m *mantarayManifest) Add(path string, entry Entry) error { + p := []byte(path) + e := entry.Reference().Bytes() + + return m.trie.Add(p, e, m.loader) +} + +func (m *mantarayManifest) Remove(path string) error { + p := []byte(path) + + err := m.trie.Remove(p, m.loader) + if err != nil { + if errors.Is(err, mantaray.ErrNotFound) { + return ErrNotFound + } + return err + } + + return nil +} + +func (m *mantarayManifest) Lookup(path string) (Entry, error) { + p := []byte(path) + + ref, err := m.trie.Lookup(p, m.loader) + if err != nil { + return nil, ErrNotFound + } + + address := swarm.NewAddress(ref) + + entry := NewEntry(address) + + return entry, nil +} + +func (m *mantarayManifest) Store(ctx context.Context, mode storage.ModePut) (swarm.Address, error) { + + saver := newMantaraySaver(ctx, m.encrypted, m.storer, mode) + + err := m.trie.Save(saver) + if err != nil { + return swarm.ZeroAddress, fmt.Errorf("manifest save error: %w", err) + } + + address := swarm.NewAddress(m.trie.Reference()) + + return address, nil +} + +// mantarayLoadSaver implements required interface 'mantaray.LoadSaver' +type mantarayLoadSaver struct { + ctx context.Context + encrypted bool + storer storage.Storer + modePut storage.ModePut +} + +func newMantarayLoader( + ctx context.Context, + encrypted bool, + storer storage.Storer, +) *mantarayLoadSaver { + return &mantarayLoadSaver{ + ctx: ctx, + encrypted: encrypted, + storer: storer, + } +} + +func newMantaraySaver( + ctx context.Context, + encrypted bool, + storer storage.Storer, + modePut storage.ModePut, +) *mantarayLoadSaver { + return &mantarayLoadSaver{ + ctx: ctx, + encrypted: encrypted, + storer: storer, + modePut: modePut, + } +} + +func (ls *mantarayLoadSaver) Load(ref []byte) ([]byte, error) { + ctx := ls.ctx + + j := joiner.NewSimpleJoiner(ls.storer) + + buf := bytes.NewBuffer(nil) + _, err := file.JoinReadAll(ctx, j, swarm.NewAddress(ref), buf, ls.encrypted) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (ls *mantarayLoadSaver) Save(data []byte) ([]byte, error) { + ctx := ls.ctx + + sp := splitter.NewSimpleSplitter(ls.storer, ls.modePut) + + address, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(data), int64(len(data)), ls.encrypted) + if err != nil { + return swarm.ZeroAddress.Bytes(), err + } + + return address.Bytes(), nil +} From e79fd9eab92ce4bce4cd841e77900e5dd79bec86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nemanja=20Zbilji=C4=87?= Date: Thu, 27 Aug 2020 12:38:33 +0200 Subject: [PATCH 2/3] Make 'mantaray' manifest default --- pkg/api/dirs_test.go | 18 +++++++++--------- pkg/api/file_test.go | 33 ++++++++++++++++++++++++++++----- pkg/api/tag_test.go | 22 +++++++++++++++++++--- pkg/manifest/manifest.go | 2 +- 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/pkg/api/dirs_test.go b/pkg/api/dirs_test.go index c0703092146..9021171d2cb 100644 --- a/pkg/api/dirs_test.go +++ b/pkg/api/dirs_test.go @@ -82,13 +82,11 @@ func TestDirs(t *testing.T) { // valid tars for _, tc := range []struct { - name string - expectedHash string - files []f // files in dir for test case + name string + files []f // files in dir for test case }{ { - name: "non-nested files without extension", - expectedHash: "685f591d0482a57e172aecb7f58babd7eb50fcb8411f875cae5c7b96fa44ff82", + name: "non-nested files without extension", files: []f{ { data: []byte("first file data"), @@ -111,8 +109,7 @@ func TestDirs(t *testing.T) { }, }, { - name: "nested files with extension", - expectedHash: "9e4e53c1764f2379408ffe019c097cbfcb8a0ba93587b52126a4e3e9d5b8556f", + name: "nested files with extension", files: []f{ { data: []byte("robots text"), @@ -166,8 +163,11 @@ func TestDirs(t *testing.T) { t.Fatal(err) } - if tc.expectedHash != resp.Reference.String() { - t.Fatalf("expected file reference to match %s, got %x", tc.expectedHash, resp.Reference) + // NOTE: cannot have expected hash for manifest because each time + // it is created it is obfuscated using random key + + if resp.Reference.String() == "" { + t.Fatalf("expected file reference, did not got any") } // read manifest metadata diff --git a/pkg/api/file_test.go b/pkg/api/file_test.go index a389758c359..ee281e86e95 100644 --- a/pkg/api/file_test.go +++ b/pkg/api/file_test.go @@ -6,6 +6,7 @@ package api_test import ( "bytes" + "encoding/json" "fmt" "io" "io/ioutil" @@ -277,7 +278,7 @@ func TestRangeRequests(t *testing.T) { uploadEndpoint: "/dirs", downloadEndpoint: "/bzz", filepath: "/ipsum/lorem.txt", - reference: "d2b1ab6fb26c1570712ca33efb30f8cbbaa994d5b85e1cf6f782bcae430eabaf", + reference: "", reader: tarFiles(t, []f{ { data: data, @@ -339,20 +340,42 @@ func TestRangeRequests(t *testing.T) { Logger: logging.New(ioutil.Discard, 5), }) + uploadReference := upload.reference + + var respBytes []byte + jsonhttptest.Request(t, client, http.MethodPost, upload.uploadEndpoint, http.StatusOK, jsonhttptest.WithRequestBody(upload.reader), - jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{ - Reference: swarm.MustParseHexAddress(upload.reference), - }), jsonhttptest.WithRequestHeader("Content-Type", upload.contentType), + jsonhttptest.WithPutResponseBody(&respBytes), ) + if uploadReference == "" { + // NOTE: cannot have expected hash for manifest because each time + // it is created it is obfuscated using random key + + read := bytes.NewReader(respBytes) + + // get the reference as everytime it will change because of random encryption key + var resp api.FileUploadResponse + err := json.NewDecoder(read).Decode(&resp) + if err != nil { + t.Fatal(err) + } + + if resp.Reference.String() == "" { + t.Fatalf("expected file reference, did not got any") + } + + uploadReference = resp.Reference.String() + } + for _, tc := range ranges { t.Run(tc.name, func(t *testing.T) { rangeHeader, want := createRangeHeader(data, tc.ranges) var body []byte - respHeaders := jsonhttptest.Request(t, client, http.MethodGet, upload.downloadEndpoint+"/"+upload.reference+upload.filepath, http.StatusPartialContent, + respHeaders := jsonhttptest.Request(t, client, http.MethodGet, upload.downloadEndpoint+"/"+uploadReference+upload.filepath, http.StatusPartialContent, jsonhttptest.WithRequestHeader("Range", rangeHeader), jsonhttptest.WithPutResponseBody(&body), ) diff --git a/pkg/api/tag_test.go b/pkg/api/tag_test.go index 09667ffc269..c77ab9aa49e 100644 --- a/pkg/api/tag_test.go +++ b/pkg/api/tag_test.go @@ -6,6 +6,7 @@ package api_test import ( "bytes" + "encoding/json" "fmt" "net/http" "strconv" @@ -460,15 +461,30 @@ func TestTags(t *testing.T) { name: "binary-file", }}) - expectedHash := swarm.MustParseHexAddress("ebcfbfac0e9a4fa4483491875f9486107a799e54cd832d0aacc59b1125b4b71f") - expectedResponse := api.FileUploadResponse{Reference: expectedHash} + var respBytes []byte respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK, jsonhttptest.WithRequestBody(tarReader), - jsonhttptest.WithExpectedJSONResponse(expectedResponse), jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar), + jsonhttptest.WithPutResponseBody(&respBytes), ) + read := bytes.NewReader(respBytes) + + // get the reference as everytime it will change because of random encryption key + var resp api.FileUploadResponse + err := json.NewDecoder(read).Decode(&resp) + if err != nil { + t.Fatal(err) + } + + // NOTE: cannot have expected hash for manifest because each time + // it is created it is obfuscated using random key + + if resp.Reference.String() == "" { + t.Fatalf("expected file reference, did not got any") + } + tagId, err := strconv.Atoi(respHeaders.Get(api.SwarmTagUidHeader)) if err != nil { t.Fatal(err) diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index a441e061e57..81da3e7e4e6 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -12,7 +12,7 @@ import ( "github.com/ethersphere/bee/pkg/swarm" ) -const DefaultManifestType = ManifestSimpleContentType +const DefaultManifestType = ManifestMantarayContentType var ( // ErrNotFound is returned when an Entry is not found in the manifest. From b94a63a1e50df6f2c8b6fcc66cce229e8b397c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nemanja=20Zbilji=C4=87?= Date: Thu, 27 Aug 2020 14:02:32 +0200 Subject: [PATCH 3/3] Update comments in tests --- pkg/api/dirs_test.go | 3 +-- pkg/api/file_test.go | 3 +-- pkg/api/tag_test.go | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/api/dirs_test.go b/pkg/api/dirs_test.go index 9021171d2cb..31bac31432f 100644 --- a/pkg/api/dirs_test.go +++ b/pkg/api/dirs_test.go @@ -163,8 +163,7 @@ func TestDirs(t *testing.T) { t.Fatal(err) } - // NOTE: cannot have expected hash for manifest because each time - // it is created it is obfuscated using random key + // NOTE: reference will be different each time, due to manifest randomness if resp.Reference.String() == "" { t.Fatalf("expected file reference, did not got any") diff --git a/pkg/api/file_test.go b/pkg/api/file_test.go index ee281e86e95..16bad394a2a 100644 --- a/pkg/api/file_test.go +++ b/pkg/api/file_test.go @@ -351,8 +351,7 @@ func TestRangeRequests(t *testing.T) { ) if uploadReference == "" { - // NOTE: cannot have expected hash for manifest because each time - // it is created it is obfuscated using random key + // NOTE: reference will be different each time, due to manifest randomness read := bytes.NewReader(respBytes) diff --git a/pkg/api/tag_test.go b/pkg/api/tag_test.go index c77ab9aa49e..e9530bba7ae 100644 --- a/pkg/api/tag_test.go +++ b/pkg/api/tag_test.go @@ -478,8 +478,7 @@ func TestTags(t *testing.T) { t.Fatal(err) } - // NOTE: cannot have expected hash for manifest because each time - // it is created it is obfuscated using random key + // NOTE: reference will be different each time, due to manifest randomness if resp.Reference.String() == "" { t.Fatalf("expected file reference, did not got any")