From 82c8bb729c181e8309403bd1854faf962d2bbee9 Mon Sep 17 00:00:00 2001 From: Bamvor Zhang Date: Thu, 16 Aug 2018 16:20:04 +0800 Subject: [PATCH 1/3] dag: add fsNodeType in NewLeafNode and NewLeafDataNode NewLeafNode and NewLeafDataNode is introduced in commit 474b77a2bdb1c ("importer: remove `UnixfsNode` from the balanced builder"). It is intended to return ipfs.Node instead of UnixfsNode. But it only support creating the TFile leaf node for merkledag. This commit add fsNodeType to above two functions and update the code in dagbuild.go. Further patches of trickledag will make use of them and pass TRaw to create leaf node. License: MIT Signed-off-by: Bamvor Zhang --- importer/balanced/builder.go | 6 +++--- importer/helpers/dagbuilder.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/importer/balanced/builder.go b/importer/balanced/builder.go index c1a3e8640..760ab320e 100644 --- a/importer/balanced/builder.go +++ b/importer/balanced/builder.go @@ -123,7 +123,7 @@ import ( func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { if db.Done() { // No data, return just an empty node. - root, err := db.NewLeafNode(nil) + root, err := db.NewLeafNode(nil, ft.TFile) if err != nil { return nil, err } @@ -137,7 +137,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { // (corner case), after that subsequent `root` nodes will // always be internal nodes (with a depth > 0) that can // be handled by the loop. - root, fileSize, err := db.NewLeafDataNode() + root, fileSize, err := db.NewLeafDataNode(ft.TFile) if err != nil { return nil, err } @@ -224,7 +224,7 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (fill if depth == 1 { // Base case: add leaf node with data. - childNode, childFileSize, err = db.NewLeafDataNode() + childNode, childFileSize, err = db.NewLeafDataNode(ft.TFile) if err != nil { return nil, 0, err } diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 24896cd1b..85c8b70aa 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -186,7 +186,7 @@ func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) { // NewLeafNode is a variation from `NewLeaf` (see its description) that // returns an `ipld.Node` instead. -func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) { +func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) { if len(data) > BlockSizeLimit { return nil, ErrSizeLimitExceeded } @@ -204,7 +204,7 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) { } // Encapsulate the data in UnixFS node (instead of a raw node). - fsNodeOverDag := db.NewFSNodeOverDag(ft.TFile) + fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType) fsNodeOverDag.SetFileData(data) node, err := fsNodeOverDag.Commit() if err != nil { @@ -273,7 +273,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { // used to keep track of the DAG file size). The size of the data is // computed here because after that it will be hidden by `NewLeafNode` // inside a generic `ipld.Node` representation. -func (db *DagBuilderHelper) NewLeafDataNode() (node ipld.Node, dataSize uint64, err error) { +func (db *DagBuilderHelper) NewLeafDataNode(fsNodeType pb.Data_DataType) (node ipld.Node, dataSize uint64, err error) { fileData, err := db.Next() if err != nil { return nil, 0, err @@ -281,7 +281,7 @@ func (db *DagBuilderHelper) NewLeafDataNode() (node ipld.Node, dataSize uint64, dataSize = uint64(len(fileData)) // Create a new leaf node containing the file chunk data. - node, err = db.NewLeafNode(fileData) + node, err = db.NewLeafNode(fileData, fsNodeType) if err != nil { return nil, 0, err } From b8a9032b3dde0edc8342c3b08a450948dfef52b4 Mon Sep 17 00:00:00 2001 From: Bamvor Zhang Date: Thu, 16 Aug 2018 16:20:16 +0800 Subject: [PATCH 2/3] Docs: update balanced builder document After fsNodeType in NewLeafNode is supported by commit 85897b3f89301 ("dag: add fsNodeType in NewLeafNode and NewLeafDataNode"). Move comments in NewLeafNode to importer/balanced/builder.go to clarify why TFile is used by balanced builder as leaves. License: MIT Signed-off-by: Bamvor Zhang --- importer/balanced/builder.go | 9 +++++++++ importer/helpers/dagbuilder.go | 5 ----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/importer/balanced/builder.go b/importer/balanced/builder.go index 760ab320e..407117dad 100644 --- a/importer/balanced/builder.go +++ b/importer/balanced/builder.go @@ -16,6 +16,15 @@ // mentioned. This is the only scenario where the root can be of a type different // that the UnixFS node. // +// Notes: +// 1. In the implementation. `FSNodeOverDag` structure is used for representing +// the UnixFS node encoded inside the DAG node. +// (see https://github.com/ipfs/go-ipfs/pull/5118.) +// 2. `TFile` is used for backwards-compatibility. It was a bug causing the leaf +// nodes to be generated with this type instead of `TRaw`. The former one +// should be used (like the trickle builder does). +// (See https://github.com/ipfs/go-ipfs/pull/5120.) +// // +-------------+ // | Root 4 | // +-------------+ diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 85c8b70aa..be381dc04 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -213,11 +213,6 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType // TODO: Encapsulate this sequence of calls into a function that // just returns the final `ipld.Node` avoiding going through // `FSNodeOverDag`. - // TODO: Using `TFile` for backwards-compatibility, a bug in the - // balanced builder was causing the leaf nodes to be generated - // with this type instead of `TRaw`, the one that should be used - // (like the trickle builder does). - // (See https://github.com/ipfs/go-ipfs/pull/5120.) return node, nil } From ab4435280a6061a76a9976bf0d148012681827d5 Mon Sep 17 00:00:00 2001 From: Bamvor Zhang Date: Thu, 16 Aug 2018 16:23:45 +0800 Subject: [PATCH 3/3] dag: remove `UnixfsNode` in Layout of trickledag This patch is the part of trickledag work which is similar to the merkledag work in commit 474b77a2bdb1c ("importer: remove `UnixfsNode` from the balanced builder"). Two helper functions(fillTrickleRecFSNode and FillFSNodeLayer) is introduced temporarily for modifing the Layout functions. These two funtions will be removed when all the code of UnixfsNode is removed in trickledag.go. Test ipfs add and get commands to check whether get the same hash of file after the code changes. License: MIT Signed-off-by: Bamvor Zhang --- importer/helpers/dagbuilder.go | 19 ++++++++++++++ importer/trickle/trickledag.go | 48 +++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index be381dc04..c411baea3 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -246,6 +246,25 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error { return nil } +// FillFSNodeLayer do the same thing as FillNodeLayer. +func (db *DagBuilderHelper) FillFSNodeLayer(node *FSNodeOverDag) error { + + // while we have room AND we're not done + for node.NumChildren() < db.maxlinks && !db.Done() { + //TODO size + child, childFileSize, err := db.NewLeafDataNode(ft.TRaw) + if err != nil { + return err + } + + if err := node.AddChild(child, childFileSize, db); err != nil { + return err + } + } + + return nil +} + // GetNextDataNode builds a UnixFsNode with the data obtained from the // Splitter, given the constraints (BlockSizeLimit, RawLeaves) specified // when creating the DagBuilderHelper. diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go index 70a953825..626044887 100644 --- a/importer/trickle/trickledag.go +++ b/importer/trickle/trickledag.go @@ -37,12 +37,13 @@ const layerRepeat = 4 // DagBuilderHelper. See the module's description for a more detailed // explanation. func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { - root := db.NewUnixfsNode() - if err := fillTrickleRec(db, root, -1); err != nil { + newRoot := db.NewFSNodeOverDag(ft.TFile) + root, _, err := fillTrickleRecFSNode(db, newRoot, -1) + if err != nil { return nil, err } - return db.AddUnixfsNode(root) + return root, db.Add(root) } // fillTrickleRec creates a trickle (sub-)tree with an optional maximum specified depth @@ -76,6 +77,47 @@ func fillTrickleRec(db *h.DagBuilderHelper, node *h.UnixfsNode, maxDepth int) er } } +// fillTrickleRecFSNode creates a trickle (sub-)tree with an optional maximum specified depth +// in the case maxDepth is greater than zero, or with unlimited depth otherwise +// (where the DAG builder will signal the end of data to end the function). +func fillTrickleRecFSNode(db *h.DagBuilderHelper, node *h.FSNodeOverDag, maxDepth int) (filledNode ipld.Node, nodeFileSize uint64, err error) { + // Always do this, even in the base case + if err := db.FillFSNodeLayer(node); err != nil { + return nil, 0, err + } + + for depth := 1; ; depth++ { + // Apply depth limit only if the parameter is set (> 0). + if db.Done() || (maxDepth > 0 && depth == maxDepth) { + break + } + for layer := 0; layer < layerRepeat; layer++ { + if db.Done() { + break + } + + nextChild := db.NewFSNodeOverDag(ft.TFile) + childNode, childFileSize, err := fillTrickleRecFSNode(db, nextChild, depth) + if err != nil { + return nil, 0, err + } + + if err := node.AddChild(childNode, childFileSize, db); err != nil { + return nil, 0, err + } + } + } + nodeFileSize = node.FileSize() + + // Get the final `dag.ProtoNode` with the `FSNode` data encoded inside. + filledNode, err = node.Commit() + if err != nil { + return nil, 0, err + } + + return filledNode, nodeFileSize, nil +} + // Append appends the data in `db` to the dag, using the Trickledag format func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out ipld.Node, errOut error) { base, ok := basen.(*dag.ProtoNode)