rclone/backend/azureblob/azureblob_internal_test.go

//go:build !plan9 && !solaris && !js

package azureblob

import (
	"context"
	"encoding/base64"
	"strings"
	"testing"

	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fstest"
	"github.com/rclone/rclone/fstest/fstests"
	"github.com/rclone/rclone/lib/random"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestBlockIDCreator(t *testing.T) {
	// Check creation and random number
	bic, err := newBlockIDCreator()
	require.NoError(t, err)
	bic2, err := newBlockIDCreator()
	require.NoError(t, err)
	assert.NotEqual(t, bic.random, bic2.random)
	assert.NotEqual(t, bic.random, [8]byte{})

	// Set random to known value for tests
	bic.random = [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
	chunkNumber := uint64(0xFEDCBA9876543210)

	// Check creation of ID
	want := base64.StdEncoding.EncodeToString([]byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, 1, 2, 3, 4, 5, 6, 7, 8})
	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", want)
	got := bic.newBlockID(chunkNumber)
	assert.Equal(t, want, got)
	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", got)

	// Test checkID is working
	assert.NoError(t, bic.checkID(chunkNumber, got))
	assert.ErrorContains(t, bic.checkID(chunkNumber, "$"+got), "illegal base64")
	assert.ErrorContains(t, bic.checkID(chunkNumber, "AAAA"+got), "bad block ID length")
	assert.ErrorContains(t, bic.checkID(chunkNumber+1, got), "expecting decoded")
	assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
}

func (f *Fs) testFeatures(t *testing.T) {
	// Check first feature flags are set on this remote
	enabled := f.Features().SetTier
	assert.True(t, enabled)
	enabled = f.Features().GetTier
	assert.True(t, enabled)
}

type ReadSeekCloser struct {
	*strings.Reader
}

func (r *ReadSeekCloser) Close() error {
	return nil
}

// Stage a block at remote but don't commit it
func (f *Fs) stageBlockWithoutCommit(ctx context.Context, t *testing.T, remote string) {
	var (
		containerName, blobPath = f.split(remote)
		containerClient         = f.cntSVC(containerName)
		blobClient              = containerClient.NewBlockBlobClient(blobPath)
		data                    = "uncommitted data"
		blockID                 = "1"
		blockIDBase64           = base64.StdEncoding.EncodeToString([]byte(blockID))
	)
	r := &ReadSeekCloser{strings.NewReader(data)}
	_, err := blobClient.StageBlock(ctx, blockIDBase64, r, nil)
	require.NoError(t, err)

	// Verify the block is staged but not committed
	blockList, err := blobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
	require.NoError(t, err)
	found := false
	for _, block := range blockList.UncommittedBlocks {
		if *block.Name == blockIDBase64 {
			found = true
			break
		}
	}
	require.True(t, found, "Block ID not found in uncommitted blocks")
}

// This tests uploading a blob where it has uncommitted blocks with a different ID size.
//
// https://gauravmantri.com/2013/05/18/windows-azure-blob-storage-dealing-with-the-specified-blob-or-block-content-is-invalid-error/
//
// TestIntegration/FsMkdir/FsPutFiles/Internal/WriteUncommittedBlocks
func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
	var (
		ctx    = context.Background()
		remote = "testBlob"
	)

	// Multipart copy the blob please
	oldUseCopyBlob, oldCopyCutoff := f.opt.UseCopyBlob, f.opt.CopyCutoff
	f.opt.UseCopyBlob = false
	f.opt.CopyCutoff = f.opt.ChunkSize
	defer func() {
		f.opt.UseCopyBlob, f.opt.CopyCutoff = oldUseCopyBlob, oldCopyCutoff
	}()

	// Create a blob with uncommitted blocks
	f.stageBlockWithoutCommit(ctx, t, remote)

	// Now attempt to overwrite the block with a different sized block ID to provoke this error

	// Check the object does not exist
	_, err := f.NewObject(ctx, remote)
	require.Equal(t, fs.ErrorObjectNotFound, err)

	// Upload a multipart file over the block with uncommitted chunks of a different ID size
	size := 4*int(f.opt.ChunkSize) - 1
	contents := random.String(size)
	item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
	o := fstests.PutTestContents(ctx, t, f, &item, contents, true)

	// Check size
	assert.Equal(t, int64(size), o.Size())

	// Create a new blob with uncommitted blocks
	newRemote := "testBlob2"
	f.stageBlockWithoutCommit(ctx, t, newRemote)

	// Copy over that block
	dst, err := f.Copy(ctx, o, newRemote)
	require.NoError(t, err)

	// Check basics
	assert.Equal(t, int64(size), dst.Size())
	assert.Equal(t, newRemote, dst.Remote())

	// Check contents
	gotContents := fstests.ReadObject(ctx, t, dst, -1)
	assert.Equal(t, contents, gotContents)

	// Remove the object
	require.NoError(t, dst.Remove(ctx))
}

func (f *Fs) InternalTest(t *testing.T) {
	t.Run("Features", f.testFeatures)
	t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
}