Import CSV File into Google Cloud SQL from Cloud Storage

I
Published in: Google Cloud - Google Apps Script

You can upload one or more CSV files to a specific bucket in Google Cloud Storage and then use Google Apps Script to import the CSV files from Cloud Storage into your Google Cloud SQL database.

In the method here, the CSV file is deleted from Cloud Storage after the import operation is complete. You can however call the /copyTo/ endpoint to move the CSV files into another Cloud Storage folder after processing.

It is important to add a wait (sleep) function because the API will throw an error if you begin uploading another file while the previous import operation is pending. The file names must be encoded as well.

function uploadtoCloudSQL() {

  // Written by Amit Agarwal amit@labnol.org
  // Web: www.ctrlq.org

  var service = getService();

  if (!service.hasAccess()) {
    Logger.log(service.getAuthorizationUrl());
    return;
  }

  var token = service.getAccessToken();

  // Getting list of files to be processed
  var result = JSON.parse(
    UrlFetchApp.fetch('https://www.googleapis.com/storage/v1/b/BUCKET_NAME/o', {
      method: "GET",
      headers: {
        Authorization: 'Bearer ' + token
      }
    }).getContentText());

  for (var i = 0; i < result.items.length; i++) {
    if (result.items[i].name.indexOf(".") !== -1) {
      files.push(result.items[i].name);
    }
  }

  for (var f = 0; f < files.length; f++) {

    var path = files[f].split("/");

    var payload =
      '{"importContext" : { "csvImportOptions": {"table":"MY_TABLE"}, "fileType": "CSV", "database": "MY_DATABASE", "uri": "gs://BUCKET_NAME/FOLDER/CSVFILE"}}'
      .replace("FOLDER", path[0])
      .replace("CSVFILE", path[1]);

    UrlFetchApp.fetch('https://www.googleapis.com/sql/v1beta4/projects/PROJECT/instances/INSTANCE/import', {
      method: "POST",
      contentType: "application/json",
      headers: {
        Authorization: 'Bearer ' + token
      },
      payload: payload,
      muteHttpExceptions: true
    });

    UrlFetchApp.fetch('https://www.googleapis.com/storage/v1/b/BUCKET_NAME/o/' + encodeURIComponent(files[f]), {
      method: "DELETE",
      headers: {
        Authorization: 'Bearer ' + token
      }
    });

    // Wait for the previous import job to end
    Utilities.sleep(5000);

  }

}
Published in: Google Cloud - Google Apps Script

Looking for something? Find here!

Meet the Author

Web Geek, Tech Columnist
A
Amit Agarwal

Amit Agarwal is a Google Developer Expert in GSuite and Google Apps Script. He holds an engineering degree in Computer Science (I.I.T.) and is the first professional blogger in India. Read more on Lifehacker and YourStory

Get in touch