How to Import CSV Files to BigQuery with Google Apps Script

August 23, 2017

I have written a Google Apps Script that will automatically upload data from one or more files in your Google Drive to your BigQuery table. This script looks for CSV file in a particular Drive Folder, uploads them to BigQuery tablet and then moves the file to another folder in Drive to indicate that it has been processed.

You need to have a table read in your BigQuery dataset and the BigQuery API needs to be enabled inside the Google Developer Console. If you are getting errors, you probably need to enable the Billing option under the BigQuery API settings. You won’t be charged but the credit cards needs to be added for you to create datasets and tables inside BigQuery.

function importCSVtoBigQuery() {
  try {
    var CSVFolder = '\\Data\\BigQuery';
    var ProcessedFolder = '\\Data\\BigQuery\\Processed';

    var projectId = '1234';
    var datasetId = 'bigquery_dataset_1';
    var tableId = 'bigquery_table_1';

    CSVFolder = getDriveFolder(CSVFolder);
    ProcessedFolder = getDriveFolder(ProcessedFolder);

    if (CSVFolder && ProcessedFolder) {
      var data,
        job,
        file,
        files = CSVFolder.getFiles();

      while (files.hasNext()) {
        file = files.next();

        if (file.getMimeType() === 'text/csv') {
          data = file.getBlob().setContentType('application/octet-stream');

          job = {
            configuration: {
              load: {
                destinationTable: {
                  projectId: projectId,
                  datasetId: datasetId,
                  tableId: tableId,
                },
                skipLeadingRows: 1,
              },
            },
          };

          job = BigQuery.Jobs.insert(job, projectId, data);
          file.makeCopy(file.getName(), ProcessedFolder);
          file.setTrashed(true);

          Logger.log('Job status for %s https://bigquery.cloud.google.com/jobs/%s', file.getName(), projectId);
        }
      }
    }
  } catch (e) {
    Logger.log(e.toString());
  }
}

// Return the ID of the Google Drive nested folder
function getDriveFolder(name) {
  var results,
    folders = name.split('\\');
  var folder = DriveApp.getRootFolder();

  for (var i = 0; i < folders.length; i++) {
    if (folders[i] === '') continue;
    results = folder.getFoldersByName(folders[i]);
    if (results.hasNext()) {
      folder = results.next();
    } else {
      folder = folder.createFolder(folders[i]);
    }
  }

  return folder;
}

BigQuery Google Apps Script

ShareWhatsAppTwitterFacebook

Amit Agarwal

Google Developer Expert, Google Cloud Champion

Amit Agarwal is a Google Developer Expert in Google Workspace and Google Apps Script. He holds an engineering degree in Computer Science (I.I.T.) and is the first professional blogger in India.

Amit has developed several popular Google add-ons including Mail Merge for Gmail and Document Studio. Read more on Lifehacker and YourStory