google cloud storage - Recursively upload a directory contents to GCS using GO SDK

Question

I am trying to upload and download a directory to GCS which container large amount of data. Can someone point me how to achieve this using Golang SDK. I would like to do multipart upload as well (-m)

Sample Directory structure:

$ tree dir1/
dir1/
└── dir2
    └── dir3
        ├── 1.csv
        └── 2.csv

Equivalent GSUtil command: gustil -m cp -r dir1 gs://exmaple/

question from:https://stackoverflow.com/questions/65603105/recursively-upload-a-directory-contents-to-gcs-using-go-sdk

Answer 1

By my understanding, I believe the GCP storage API is atomic and only allows for a single object upload at a time.

The gsutil command uses recursion to iterate through the provided path but ultimately does singular API calls for uploading.

The snippet below will give you some hints, please not I haven't tested the actual copying to GCP. I have only verified that the code compiles.

It's a modified version of what Google have in their docs Uploading Objects

package main

import (
    "context"
    "fmt"
    "io"
    "os"
    "path/filepath"
    "strings"
    "time"

    "cloud.google.com/go/storage"
)

// BulkUpload uploads files in bulk
func BulkUpload(bucket, rootPath string) error {
    ctx := context.Background()
    client, err := storage.NewClient(ctx)
    if err != nil {
        return fmt.Errorf("storage.NewClient: %v", err)
    }
    defer client.Close()

    fileList, objPath, err := pathWalk(rootPath)
    if err != nil {
        return err
    }
    _ = objPath

    for i:=0; i <len(fileList);i++ {
        // Open and read local file
        f, err := os.Open(fileList[i])
        if err != nil {
            return fmt.Errorf("os.Open: %v", err)
        }
        f.Close()

        ctx, cancel := context.WithTimeout(ctx, time.Second*50)
        defer cancel()

        // Upload an object with storage.Writer.
        wc := client.Bucket(bucket).Object(objPath[i]).NewWriter(ctx)
        if _, err = io.Copy(wc, f); err != nil {
            return fmt.Errorf("io.Copy: %v", err)
        }
        if err := wc.Close(); err != nil {
            return fmt.Errorf("Writer.Close: %v", err)
        }
        fmt.Printf("Blob %v uploaded.
", objPath[i])
    }

    return nil
}


// pathWalk create a list of files from a root path
// returns a list of all files and a list of files excluding the root path
func pathWalk(root string) ([]string, []string, error){
    var files []string
    var objectPath []string

    err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
        files = append(files, path)
        return nil
    })
    if err != nil {
        return nil, nil, fmt.Errorf("error walking path: %v", err)
    }

    for _, f := range files {
        objectPath = append(objectPath, strings.Trim(f, root))
    }

    return files, objectPath, nil
}

func main() {

    if err := BulkUpload("mybucket", "somedirectory/" ); err != nil {
        panic(err)
    }

}