[WIP] remove UnixfsNode from trickledag

importer/balanced/builder.go

@@ -16,6 +16,15 @@
 // mentioned. This is the only scenario where the root can be of a type different
 // that the UnixFS node.
 //
+// Notes:
+// 1. In the implementation. `FSNodeOverDag` structure is used for representing
+//    the UnixFS node encoded inside the DAG node.
+//    (see https://github.com/ipfs/go-ipfs/pull/5118.)
+// 2. `TFile` is used for backwards-compatibility. It was a bug causing the leaf
+//    nodes to be generated with this type instead of `TRaw`. The former one
+//    should be used (like the trickle builder does).
+//    (See https://github.com/ipfs/go-ipfs/pull/5120.)
+//
 //                                                 +-------------+
 //                                                 |   Root 4    |
 //                                                 +-------------+
@@ -123,7 +132,7 @@ import (
 func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
 	if db.Done() {
 		// No data, return just an empty node.
-		root, err := db.NewLeafNode(nil)
+		root, err := db.NewLeafNode(nil, ft.TFile)
 		if err != nil {
 			return nil, err
 		}
@@ -137,7 +146,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
 	// (corner case), after that subsequent `root` nodes will
 	// always be internal nodes (with a depth > 0) that can
 	// be handled by the loop.
-	root, fileSize, err := db.NewLeafDataNode()
+	root, fileSize, err := db.NewLeafDataNode(ft.TFile)
 	if err != nil {
 		return nil, err
 	}
@@ -224,7 +233,7 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (fill
 
 		if depth == 1 {
 			// Base case: add leaf node with data.
-			childNode, childFileSize, err = db.NewLeafDataNode()
+			childNode, childFileSize, err = db.NewLeafDataNode(ft.TFile)
 			if err != nil {
 				return nil, 0, err
 			}

importer/helpers/dagbuilder.go

@@ -186,7 +186,7 @@ func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) {
 
 // NewLeafNode is a variation from `NewLeaf` (see its description) that
 // returns an `ipld.Node` instead.
-func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) {
+func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) {
 	if len(data) > BlockSizeLimit {
 		return nil, ErrSizeLimitExceeded
 	}
@@ -204,7 +204,7 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) {
 	}
 
 	// Encapsulate the data in UnixFS node (instead of a raw node).
-	fsNodeOverDag := db.NewFSNodeOverDag(ft.TFile)
+	fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType)
 	fsNodeOverDag.SetFileData(data)
 	node, err := fsNodeOverDag.Commit()
 	if err != nil {
@@ -213,11 +213,6 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) {
 	// TODO: Encapsulate this sequence of calls into a function that
 	// just returns the final `ipld.Node` avoiding going through
 	// `FSNodeOverDag`.
-	// TODO: Using `TFile` for backwards-compatibility, a bug in the
-	// balanced builder was causing the leaf nodes to be generated
-	// with this type instead of `TRaw`, the one that should be used
-	// (like the trickle builder does).
-	// (See https://github.com/ipfs/go-ipfs/pull/5120.)
 
 	return node, nil
 }
@@ -251,6 +246,25 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error {
 	return nil
 }
 
+// FillFSNodeLayer do the same thing as FillNodeLayer.
+func (db *DagBuilderHelper) FillFSNodeLayer(node *FSNodeOverDag) error {
+
+	// while we have room AND we're not done
+	for node.NumChildren() < db.maxlinks && !db.Done() {
+		//TODO size
+		child, childFileSize, err := db.NewLeafDataNode(ft.TRaw)
+		if err != nil {
+			return err
+		}
+
+		if err := node.AddChild(child, childFileSize, db); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // GetNextDataNode builds a UnixFsNode with the data obtained from the
 // Splitter, given the constraints (BlockSizeLimit, RawLeaves) specified
 // when creating the DagBuilderHelper.
@@ -273,15 +287,15 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
 // used to keep track of the DAG file size). The size of the data is
 // computed here because after that it will be hidden by `NewLeafNode`
 // inside a generic `ipld.Node` representation.
-func (db *DagBuilderHelper) NewLeafDataNode() (node ipld.Node, dataSize uint64, err error) {
+func (db *DagBuilderHelper) NewLeafDataNode(fsNodeType pb.Data_DataType) (node ipld.Node, dataSize uint64, err error) {
 	fileData, err := db.Next()
 	if err != nil {
 		return nil, 0, err
 	}
 	dataSize = uint64(len(fileData))
 
 	// Create a new leaf node containing the file chunk data.
-	node, err = db.NewLeafNode(fileData)
+	node, err = db.NewLeafNode(fileData, fsNodeType)
 	if err != nil {
 		return nil, 0, err
 	}

importer/trickle/trickledag.go

@@ -37,12 +37,13 @@ const layerRepeat = 4
 // DagBuilderHelper. See the module's description for a more detailed
 // explanation.
 func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
-	root := db.NewUnixfsNode()
-	if err := fillTrickleRec(db, root, -1); err != nil {
+	newRoot := db.NewFSNodeOverDag(ft.TFile)
+	root, _, err := fillTrickleRecFSNode(db, newRoot, -1)
+	if err != nil {
 		return nil, err
 	}
 
-	return db.AddUnixfsNode(root)
+	return root, db.Add(root)
 }
 
 // fillTrickleRec creates a trickle (sub-)tree with an optional maximum specified depth
@@ -76,6 +77,47 @@ func fillTrickleRec(db *h.DagBuilderHelper, node *h.UnixfsNode, maxDepth int) er
 	}
 }
 
+// fillTrickleRecFSNode creates a trickle (sub-)tree with an optional maximum specified depth
+// in the case maxDepth is greater than zero, or with unlimited depth otherwise
+// (where the DAG builder will signal the end of data to end the function).
+func fillTrickleRecFSNode(db *h.DagBuilderHelper, node *h.FSNodeOverDag, maxDepth int) (filledNode ipld.Node, nodeFileSize uint64, err error) {
+	// Always do this, even in the base case
+	if err := db.FillFSNodeLayer(node); err != nil {
+		return nil, 0, err
+	}
+
+	for depth := 1; ; depth++ {
+		// Apply depth limit only if the parameter is set (> 0).
+		if db.Done() || (maxDepth > 0 && depth == maxDepth) {
+			break
+		}
+		for layer := 0; layer < layerRepeat; layer++ {
+			if db.Done() {
+				break
+			}
+
+			nextChild := db.NewFSNodeOverDag(ft.TFile)
+			childNode, childFileSize, err := fillTrickleRecFSNode(db, nextChild, depth)
+			if err != nil {
+				return nil, 0, err
+			}
+
+			if err := node.AddChild(childNode, childFileSize, db); err != nil {
+				return nil, 0, err
+			}
+		}
+	}
+	nodeFileSize = node.FileSize()
+
+	// Get the final `dag.ProtoNode` with the `FSNode` data encoded inside.
+	filledNode, err = node.Commit()
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return filledNode, nodeFileSize, nil
+}
+
 // Append appends the data in `db` to the dag, using the Trickledag format
 func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out ipld.Node, errOut error) {
 	base, ok := basen.(*dag.ProtoNode)