How to Import CSV Files to BigQuery with Google Apps Script

H
Published in: BigQuery - Google Apps Script

I have written a Google Apps Script that will automatically upload data from one or more files in your Google Drive to your BigQuery table. This script looks for CSV file in a particular Drive Folder, uploads them to BigQuery tablet and then moves the file to another folder in Drive to indicate that it has been processed.

You need to have a table read in your BigQuery dataset and the BigQuery API needs to be enabled inside the Google Developer Console. If you are getting errors, you probably need to enable the Billing option under the BigQuery API settings. You won’t be charged but the credit cards needs to be added for you to create datasets and tables inside BigQuery.

function importCSVtoBigQuery() {

  try {

  var CSVFolder = "\\Data\\BigQuery";
  var ProcessedFolder = "\\Data\\BigQuery\\Processed";

  var projectId = '1234';
  var datasetId = 'bigquery_dataset_1';
  var tableId   = 'bigquery_table_1';

  CSVFolder = getDriveFolder(CSVFolder);
  ProcessedFolder = getDriveFolder(ProcessedFolder);

  if (CSVFolder && ProcessedFolder) {

    var data, job, file, files = CSVFolder.getFiles();

    while (files.hasNext()) {

      file = files.next();

      if (file.getMimeType() === "text/csv") {

        data = file.getBlob().setContentType('application/octet-stream');

        job = {
          configuration: {
            load: {
              destinationTable: {
                projectId: projectId,
                datasetId: datasetId,
                tableId: tableId
              },
              skipLeadingRows: 1
            }
          }
        };

        job = BigQuery.Jobs.insert(job, projectId, data);
        file.makeCopy(file.getName(), ProcessedFolder);
        file.setTrashed(true);

        Logger.log('Job status for %s https://bigquery.cloud.google.com/jobs/%s', file.getName(), projectId);
      }
    }

  }
  } catch(e) {
    Logger.log(e.toString());
  }
}

// Return the ID of the Google Drive nested folder
function getDriveFolder(name) {

  var results, folders = name.split("\\");
  var folder = DriveApp.getRootFolder();

  for (var i=0; i<folders.length; i++) {
    if (folders[i] === "") continue;
    results = folder.getFoldersByName(folders[i]);
    if (results.hasNext()) {
      folder = results.next();
    } else {
      folder = folder.createFolder(folders[i]);
    }
  }

  return folder;
}
Published in: BigQuery - Google Apps Script

Looking for something? Find here!

Meet the Author

Web Geek, Tech Columnist
A
Amit Agarwal

Amit Agarwal is a Google Developer Expert in GSuite and Google Apps Script. He holds an engineering degree in Computer Science (I.I.T.) and is the first professional blogger in India. Read more on Lifehacker and YourStory

Get in touch