Merge pull request #3323 from filecoin-project/steb/fix-post

Fix PoSt with bad sectors

Author: magik6k

api/test/util.go

@@ -41,13 +41,16 @@ func MineUntilBlock(ctx context.Context, t *testing.T, sn TestStorageNode, cb fu
 		var success bool
 		var err error
 		wait := make(chan struct{})
-		sn.MineOne(ctx, miner.MineReq{
+		mineErr := sn.MineOne(ctx, miner.MineReq{
 			Done: func(win bool, e error) {
 				success = win
 				err = e
 				wait <- struct{}{}
 			},
 		})
+		if mineErr != nil {
+			t.Fatal(mineErr)
+		}
 		<-wait
 		if err != nil {
 			t.Fatal(err)

api/test/window_post.go

@@ -216,6 +216,10 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		sn, err := parts[0].Sectors.First()
 		require.NoError(t, err)
 
+		all, err := parts[0].Sectors.All(2)
+		require.NoError(t, err)
+		fmt.Println("the sectors", all)
+
 		s = abi.SectorID{
 			Miner:  abi.ActorID(mid),
 			Number: abi.SectorNumber(sn),

extern/sector-storage/mock/mock.go

@@ -3,12 +3,12 @@ package mock
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"math/rand"
 	"sync"
 
-	"github.com/filecoin-project/go-bitfield"
 	commcid "github.com/filecoin-project/go-fil-commcid"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-storage/storage"
@@ -291,32 +291,29 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI
 	return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil
 }
 
-func generateFakePoSt(sectorInfo []abi.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []abi.PoStProof {
-	sectors := bitfield.New()
+func generateFakePoStProof(sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) []byte {
+	hasher := sha256.New()
+	_, _ = hasher.Write(randomness)
 	for _, info := range sectorInfo {
-		sectors.Set(uint64(info.SectorNumber))
-	}
-
-	wp, err := rpt(sectorInfo[0].SealProof)
-	if err != nil {
-		panic(err)
+		err := info.MarshalCBOR(hasher)
+		if err != nil {
+			panic(err)
+		}
 	}
+	return hasher.Sum(nil)
 
-	var proofBuf bytes.Buffer
+}
 
-	_, err = proofBuf.Write(randomness)
+func generateFakePoSt(sectorInfo []abi.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []abi.PoStProof {
+	wp, err := rpt(sectorInfo[0].SealProof)
 	if err != nil {
 		panic(err)
 	}
 
-	if err := sectors.MarshalCBOR(&proofBuf); err != nil {
-		panic(err)
-	}
-
 	return []abi.PoStProof{
 		{
 			PoStProof:  wp,
-			ProofBytes: proofBuf.Bytes(),
+			ProofBytes: generateFakePoStProof(sectorInfo, randomness),
 		},
 	}
 }
@@ -412,36 +409,10 @@ func (m mockVerif) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVeri
 
 	proof := info.Proofs[0]
 
-	if !bytes.Equal(proof.ProofBytes[:len(info.Randomness)], info.Randomness) {
-		return false, xerrors.Errorf("bad randomness")
+	expected := generateFakePoStProof(info.ChallengedSectors, info.Randomness)
+	if !bytes.Equal(proof.ProofBytes, expected) {
+		return false, xerrors.Errorf("bad proof")
 	}
-
-	sectors := bitfield.New()
-	if err := sectors.UnmarshalCBOR(bytes.NewReader(proof.ProofBytes[len(info.Randomness):])); err != nil {
-		return false, xerrors.Errorf("unmarshaling sectors bitfield from \"proof\": %w", err)
-	}
-
-	challenged := bitfield.New()
-	for _, sector := range info.ChallengedSectors {
-		challenged.Set(uint64(sector.SectorNumber))
-	}
-
-	{
-		b1, err := sectors.MarshalJSON()
-		if err != nil {
-			return false, err
-		}
-
-		b2, err := challenged.MarshalJSON()
-		if err != nil {
-			return false, err
-		}
-
-		if !bytes.Equal(b1, b2) {
-			return false, xerrors.Errorf("proven and challenged sector sets didn't match: %s != %s", string(b1), string(b2))
-		}
-	}
-
 	return true, nil
 }
 

storage/wdpost_run.go

@@ -366,7 +366,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
 
 		skipCount += sc
 
-		ssi, err := s.sectorInfo(ctx, good, ts)
+		ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts)
 		if err != nil {
 			return nil, xerrors.Errorf("getting sorted sector info: %w", err)
 		}
@@ -399,8 +399,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
 
 	tsStart := build.Clock.Now()
 
-	log.Infow("generating windowPost", "sectors", len(sinfos))
-
 	mid, err := address.IDFromAddress(s.actor)
 	if err != nil {
 		return nil, err
@@ -436,22 +434,44 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo
 	return params, nil
 }
 
-func (s *WindowPoStScheduler) sectorInfo(ctx context.Context, deadlineSectors abi.BitField, ts *types.TipSet) ([]abi.SectorInfo, error) {
-	sset, err := s.api.StateMinerSectors(ctx, s.actor, &deadlineSectors, false, ts.Key())
+func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors abi.BitField, ts *types.TipSet) ([]abi.SectorInfo, error) {
+	sset, err := s.api.StateMinerSectors(ctx, s.actor, &goodSectors, false, ts.Key())
 	if err != nil {
 		return nil, err
 	}
 
-	sbsi := make([]abi.SectorInfo, len(sset))
-	for k, sector := range sset {
-		sbsi[k] = abi.SectorInfo{
+	if len(sset) == 0 {
+		return nil, nil
+	}
+
+	substitute := abi.SectorInfo{
+		SectorNumber: sset[0].ID,
+		SealedCID:    sset[0].Info.SealedCID,
+		SealProof:    sset[0].Info.SealProof,
+	}
+
+	sectorByID := make(map[uint64]abi.SectorInfo, len(sset))
+	for _, sector := range sset {
+		sectorByID[uint64(sector.ID)] = abi.SectorInfo{
 			SectorNumber: sector.ID,
 			SealedCID:    sector.Info.SealedCID,
 			SealProof:    sector.Info.SealProof,
 		}
 	}
 
-	return sbsi, nil
+	proofSectors := make([]abi.SectorInfo, 0, len(sset))
+	if err := allSectors.ForEach(func(sectorNo uint64) error {
+		if info, found := sectorByID[sectorNo]; found {
+			proofSectors = append(proofSectors, info)
+		} else {
+			proofSectors = append(proofSectors, substitute)
+		}
+		return nil
+	}); err != nil {
+		return nil, xerrors.Errorf("iterating partition sector bitmap: %w", err)
+	}
+
+	return proofSectors, nil
 }
 
 func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.SubmitWindowedPoStParams) error {