How to Import CSV Files to BigQuery with Google Apps Script

H
Published in: BigQuery - Google Apps Script

I have written a Google Apps Script that will automatically upload data from one or more files in your Google Drive to your BigQuery table. This script looks for CSV file in a particular Drive Folder, uploads them to BigQuery tablet and then moves the file to another folder in Drive to indicate that it has been processed.

You need to have a table read in your BigQuery dataset and the BigQuery API needs to be enabled inside the Google Developer Console. If you are getting errors, you probably need to enable the Billing option under the BigQuery API settings. You won’t be charged but the credit cards needs to be added for you to create datasets and tables inside BigQuery.

function importCSVtoBigQuery() {

  try {

  var CSVFolder = "\\Data\\BigQuery";
  var ProcessedFolder = "\\Data\\BigQuery\\Processed";

  var projectId = '1234';
  var datasetId = 'bigquery_dataset_1';
  var tableId   = 'bigquery_table_1';

  CSVFolder = getDriveFolder(CSVFolder);
  ProcessedFolder = getDriveFolder(ProcessedFolder);

  if (CSVFolder && ProcessedFolder) {

    var data, job, file, files = CSVFolder.getFiles();

    while (files.hasNext()) {

      file = files.next();

      if (file.getMimeType() === "text/csv") {

        data = file.getBlob().setContentType('application/octet-stream');

        job = {
          configuration: {
            load: {
              destinationTable: {
                projectId: projectId,
                datasetId: datasetId,
                tableId: tableId
              },
              skipLeadingRows: 1
            }
          }
        };

        job = BigQuery.Jobs.insert(job, projectId, data);
        file.makeCopy(file.getName(), ProcessedFolder);
        file.setTrashed(true);

        Logger.log('Job status for %s https://bigquery.cloud.google.com/jobs/%s', file.getName(), projectId);
      }
    }

  }
  } catch(e) {
    Logger.log(e.toString());
  }
}

// Return the ID of the Google Drive nested folder
function getDriveFolder(name) {

  var results, folders = name.split("\\");
  var folder = DriveApp.getRootFolder();

  for (var i=0; i<folders.length; i++) {
    if (folders[i] === "") continue;
    results = folder.getFoldersByName(folders[i]);
    if (results.hasNext()) {
      folder = results.next();
    } else {
      folder = folder.createFolder(folders[i]);
    }
  }

  return folder;
}
📮  Subscribe to our Email Newsletter for Google tips and tutorials!
Published in: BigQuery - Google Apps Script

Looking for something? Find here!

Meet the Author

Web Geek, Google Developer Expert
A
Amit Agarwal

Amit Agarwal is a Google Developer Expert in Google Workspace and Google Apps Script. He holds an engineering degree in Computer Science (I.I.T.) and is the first professional blogger in India. He is the developer of Mail Merge for Gmail and Document Studio. Read more on Lifehacker and YourStory

Get in touch

Google Add-ons

Do more with your Gmail and GSuite account
G

We build bespoke solutions that use the capabilities and the features of Google Workspace for automating business processes and driving work productivity.

  1. Mail Merge with Attachments
    Send personalized email to your Google Contact with a Google Sheet and Gmail
  2. Save Emails and Attachments
    Download email messages and file attachments from Gmail to your Google Drive
  3. Google Forms Email Notifications
    Send email notifications to multiple people when a new Google Form is submitted
  4. Document Studio
    Create beautiful pixel perfect documents merging data from Google Sheets and Google Forms
  5. Creator Studio for Google Slides
    Turn your Google Slides presentations into animated GIFs and videos for uploading to YouTube