feat: Checksums - add Checksums and Digest types

internal/pkg/metadb/checksums/checksums.go

@@ -0,0 +1,79 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checksums
+
+import (
+	"bytes"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// Checksums is a struct for blob checksums.
+type Checksums struct {
+	// MD5 is the MD5 hash value of the associated blob object.
+	MD5 []byte `datastore:",noindex"`
+	// CRC32C is the CRC32C checksum of the associated blob object.
+	// CRC32C uses the Castagnoli polynomial.
+	// This needs to be int32 because Datastore doesn't support unsigned integers...
+	// Use SetCRC32C() and GetCRC32C() for easier access.
+	CRC32C int32 `datastore:",noindex"`
+	// HasCRC32C indicates if there is a valid CRC32C value.
+	HasCRC32C bool `datastore:",noindex"`
+}
+
+// SetCRC32C sets v to CRC32C and sets HasCRC32C to true.
+func (c *Checksums) SetCRC32C(v uint32) {
+	c.HasCRC32C = true
+	c.CRC32C = int32(v)
+}
+
+// GetCRC32C returns CRC32C.
+func (c *Checksums) GetCRC32C() uint32 {
+	return uint32(c.CRC32C)
+}
+
+// ResetCRC32C clears the CRC32C value and sets HasCRC32C to false.
+func (c *Checksums) ResetCRC32C() {
+	c.CRC32C = 0 // 0 is a possible CRC32C value but would still help debugging.
+	c.HasCRC32C = false
+}
+
+// ChecksumsProto represents an proto message that has checksum values.
+type ChecksumsProto interface {
+	GetHasCrc32C() bool
+	GetCrc32C() uint32
+	GetMd5() []byte
+}
+
+// ValidateIfPresent checks if the checksum values match with p.
+// Returns a DataLoss error if they don't.
+func (c *Checksums) ValidateIfPresent(p ChecksumsProto) error {
+	if len(p.GetMd5()) != 0 && len(c.MD5) != 0 {
+		if !bytes.Equal(p.GetMd5(), c.MD5) {
+			return status.Errorf(codes.DataLoss,
+				"MD5 hash values didn't match: provided[%v], calculated[%v]",
+				p.GetMd5(), c.MD5)
+		}
+	}
+	if p.GetHasCrc32C() && c.HasCRC32C {
+		if p.GetCrc32C() != c.GetCRC32C() {
+			return status.Errorf(codes.DataLoss,
+				"CRC32C checksums didn't match: provided[%v], calculated[%v]",
+				p.GetCrc32C(), p.GetCrc32C())
+		}
+	}
+	return nil
+}

internal/pkg/metadb/checksums/checksums_test.go

@@ -0,0 +1,76 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checksums_test
+
+import (
+	"crypto/md5"
+	"testing"
+
+	"github.com/googleforgames/open-saves/internal/pkg/metadb/checksums"
+	"github.com/googleforgames/open-saves/internal/pkg/metadb/checksums/checksumstest"
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestChecksums_CRC32C(t *testing.T) {
+	var c checksums.Checksums
+	c.SetCRC32C(uint32(0xfedcba98))
+	assert.Equal(t, uint32(0xfedcba98), c.GetCRC32C())
+	c.ResetCRC32C()
+	assert.False(t, c.HasCRC32C)
+	assert.Zero(t, c.CRC32C)
+}
+
+func TestChecksums_ValidateIfPresent(t *testing.T) {
+	c := checksumstest.RandomChecksums(t)
+	none := new(checksumstest.ChecksumsProtoImpl)
+	assert.NoError(t, c.ValidateIfPresent(none))
+	md5 := make([]byte, md5.Size)
+	copy(md5, c.MD5)
+
+	hasMD5 := &checksumstest.ChecksumsProtoImpl{
+		Checksums: checksums.Checksums{
+			MD5: md5,
+		},
+	}
+	assert.NoError(t, c.ValidateIfPresent(hasMD5))
+
+	hasCRC32C := &checksumstest.ChecksumsProtoImpl{
+		Checksums: checksums.Checksums{
+			CRC32C:    c.CRC32C,
+			HasCRC32C: true,
+		},
+	}
+	assert.NoError(t, c.ValidateIfPresent(hasCRC32C))
+
+	hasBoth := &checksumstest.ChecksumsProtoImpl{
+		Checksums: checksums.Checksums{
+			MD5:       md5,
+			CRC32C:    c.CRC32C,
+			HasCRC32C: true,
+		},
+	}
+	assert.NoError(t, c.ValidateIfPresent(hasBoth))
+
+	// Tweak values so they don't match
+	c.MD5[0] += 1
+	c.CRC32C += 1
+
+	assert.NoError(t, c.ValidateIfPresent(none))
+	assert.Equal(t, codes.DataLoss, status.Code(c.ValidateIfPresent(hasMD5)))
+	assert.Equal(t, codes.DataLoss, status.Code(c.ValidateIfPresent(hasCRC32C)))
+	assert.Equal(t, codes.DataLoss, status.Code(c.ValidateIfPresent(hasBoth)))
+}

internal/pkg/metadb/checksums/checksumstest/checksumstest.go

@@ -0,0 +1,79 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checksumstest
+
+import (
+	"crypto/md5"
+	"math/rand"
+	"testing"
+
+	"cloud.google.com/go/datastore"
+	"github.com/googleforgames/open-saves/internal/pkg/metadb/checksums"
+	"github.com/stretchr/testify/assert"
+)
+
+// AssertPropertyListMatch asserts the properties in actual has the same values to expected.
+func AssertPropertyListMatch(t *testing.T, expected checksums.Checksums, actual []datastore.Property) {
+	t.Helper()
+	ps, err := datastore.SaveStruct(&expected)
+	if assert.NoError(t, err) {
+		assert.ElementsMatch(t, ps, actual)
+	}
+}
+
+// ChecksumsToProperties creates a list of datastore.Property with values in c.
+func ChecksumsToProperties(t *testing.T, c checksums.Checksums) []datastore.Property {
+	t.Helper()
+	ps, err := datastore.SaveStruct(&c)
+	assert.NoError(t, err)
+	return ps
+}
+
+// RandomChecksums returns a randomly initialized Checksums for tests.
+func RandomChecksums(t *testing.T) checksums.Checksums {
+	t.Helper()
+	md5 := make([]byte, md5.Size)
+	rand.Read(md5)
+	return checksums.Checksums{
+		MD5:       md5,
+		CRC32C:    int32(rand.Uint32()),
+		HasCRC32C: true,
+	}
+}
+
+// AssertProtoEquals checks actual has the same values as expected.
+func AssertProtoEqual(t *testing.T, expected checksums.Checksums, actual checksums.ChecksumsProto) {
+	t.Helper()
+	assert.Equal(t, expected.MD5, actual.GetMd5())
+	assert.Equal(t, expected.GetCRC32C(), actual.GetCrc32C())
+	assert.Equal(t, expected.HasCRC32C, actual.GetHasCrc32C())
+}
+
+// ChecksumsProtoImpl is a wrapper of Checksums for tests.
+type ChecksumsProtoImpl struct {
+	checksums.Checksums
+}
+
+func (c *ChecksumsProtoImpl) GetMd5() []byte {
+	return c.MD5
+}
+
+func (c *ChecksumsProtoImpl) GetCrc32C() uint32 {
+	return c.GetCRC32C()
+}
+
+func (c *ChecksumsProtoImpl) GetHasCrc32C() bool {
+	return c.HasCRC32C
+}

internal/pkg/metadb/checksums/checksumstest/checksumstest_test.go

@@ -0,0 +1,45 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checksumstest
+
+import (
+	"crypto/md5"
+	"testing"
+
+	"cloud.google.com/go/datastore"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChecksumsTest_Random(t *testing.T) {
+	c := RandomChecksums(t)
+	if assert.NotNil(t, c) {
+		assert.Len(t, c.MD5, md5.Size)
+		assert.True(t, c.HasCRC32C)
+	}
+}
+
+func TestChecksumsTest_AssertPropertyListMatch(t *testing.T) {
+	c := RandomChecksums(t)
+	ps, err := datastore.SaveStruct(&c)
+	if assert.NoError(t, err) {
+		AssertPropertyListMatch(t, c, ps)
+	}
+}
+
+func TestChecksumsTest_AssertProtoEqual(t *testing.T) {
+	c := RandomChecksums(t)
+	p := &ChecksumsProtoImpl{Checksums: c}
+	AssertProtoEqual(t, c, p)
+}

internal/pkg/metadb/checksums/digest.go

@@ -0,0 +1,70 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package checksums
+
+import (
+	"crypto/md5"
+	"hash"
+	"hash/crc32"
+	"io"
+	"sync"
+)
+
+var crc32cTable *crc32.Table
+var tableInitOnce = new(sync.Once)
+
+// Digest calculates MD5 and CRC32C values as you call Write and
+// returns the values as a Checksums variable.
+type Digest struct {
+	md5    hash.Hash
+	crc32c hash.Hash32
+}
+
+// Assert that Digest implements io.Writer.
+var _ io.Writer = new(Digest)
+
+// NewDigest creates a new instance of Digest.
+func NewDigest() *Digest {
+	tableInitOnce.Do(func() { crc32cTable = crc32.MakeTable(crc32.Castagnoli) })
+	return &Digest{
+		md5:    md5.New(),
+		crc32c: crc32.New(crc32cTable),
+	}
+}
+
+// Write adds more data to the running hashes.
+// It never returns an error.
+func (d *Digest) Write(p []byte) (int, error) {
+	// Write never returns an error for Hash objects.
+	// https://pkg.go.dev/hash#Hash
+	d.crc32c.Write(p)
+	d.md5.Write(p)
+	return len(p), nil
+}
+
+// Reset resets the hash states.
+func (d *Digest) Reset() {
+	d.md5.Reset()
+	d.crc32c.Reset()
+}
+
+// Checksums returns a new Checksums variable with the calculated hash values.
+func (d *Digest) Checksums() Checksums {
+	return Checksums{
+		MD5:       d.md5.Sum(nil),
+		CRC32C:    int32(d.crc32c.Sum32()),
+		HasCRC32C: true,
+	}
+}