client/client.go
func (c *Client) largeDataPagedQuery(service *bigquery.Service, pageSize int, dataset, project, queryStr string, dataChan chan Data) ([][]interface{}, []string, error) {
	c.printDebug("largeDataPagedQuery starting")
	ts := time.Now()
	// start query
	tableRef := bigquery.TableReference{DatasetId: dataset, ProjectId: project, TableId: c.tempTableName}
	jobConfigQuery := bigquery.JobConfigurationQuery{}

	datasetRef := &bigquery.DatasetReference{
		DatasetId: dataset,
		ProjectId: project,
	}

	jobConfigQuery.AllowLargeResults = true
	jobConfigQuery.Query = queryStr
	jobConfigQuery.DestinationTable = &tableRef
	jobConfigQuery.DefaultDataset = datasetRef
	if !c.flattenResults {
		c.printDebug("setting FlattenResults to false")
		// need a pointer to bool
		f := false
		jobConfigQuery.FlattenResults = &f
	}
	jobConfigQuery.WriteDisposition = "WRITE_TRUNCATE"
	jobConfigQuery.CreateDisposition = "CREATE_IF_NEEDED"

	jobConfig := bigquery.JobConfiguration{}

	jobConfig.Query = &jobConfigQuery

	job := bigquery.Job{}
	job.Configuration = &jobConfig

	jobInsert := service.Jobs.Insert(project, &job)
	runningJob, jerr := jobInsert.Do()

	if jerr != nil {
		c.printDebug("Error inserting job!", jerr)
		if dataChan != nil {
			dataChan <- Data{Err: jerr}
		}
		return nil, nil, jerr
	}

	var qr *bigquery.GetQueryResultsResponse
	var rows [][]interface{}
	var headers []string
	var err error

	// Periodically, job references are not created, but errors are also not thrown.
	// In this scenario, retry up to 5 times to get a job reference before giving up.
	for i := 1; ; i++ {
		r := service.Jobs.GetQueryResults(project, runningJob.JobReference.JobId)
		r.TimeoutMs(c.RequestTimeout)
		qr, err = r.Do()

		headers, rows = c.headersAndRows(qr.Schema, qr.Rows)

		if i >= maxRequestRetry || qr.JobReference != nil || err != nil {
			if i > 1 {
				c.printDebug("Took %v tries to get a job reference", i)
			}
			break
		}
	}

	if err == nil && qr.JobReference == nil {
		err = fmt.Errorf("missing job reference")
	}

	if err != nil {
		c.printDebug("Error loading query: ", err)
		if dataChan != nil {
			dataChan <- Data{Err: err}
		}
		return nil, nil, err
	}

	rows, headers, err = c.processPagedQuery(qr.JobReference, qr.PageToken, dataChan, headers, rows)
	c.printDebug("largeDataPagedQuery completed in ", time.Now().Sub(ts).Seconds(), "s")

	return rows, headers, err
}
func (c *Client) largeDataPagedQuery(service *bigquery.Service, pageSize int, dataset, project, queryStr string, dataChan chan Data) ([][]interface{}, []string, error) {
	c.printDebug("largeDataPagedQuery starting")
	ts := time.Now()
	// start query
	tableRef := bigquery.TableReference{DatasetId: dataset, ProjectId: project, TableId: c.tempTableName}
	jobConfigQuery := bigquery.JobConfigurationQuery{}

	datasetRef := &bigquery.DatasetReference{
		DatasetId: dataset,
		ProjectId: project,
	}

	jobConfigQuery.AllowLargeResults = true
	jobConfigQuery.Query = queryStr
	jobConfigQuery.DestinationTable = &tableRef
	jobConfigQuery.DefaultDataset = datasetRef
	if !c.flattenResults {
		c.printDebug("setting FlattenResults to false")
		// need a pointer to bool
		f := false
		jobConfigQuery.FlattenResults = &f
	}
	jobConfigQuery.WriteDisposition = "WRITE_TRUNCATE"
	jobConfigQuery.CreateDisposition = "CREATE_IF_NEEDED"

	jobConfig := bigquery.JobConfiguration{}

	jobConfig.Query = &jobConfigQuery

	job := bigquery.Job{}
	job.Configuration = &jobConfig

	jobInsert := service.Jobs.Insert(project, &job)
	runningJob, jerr := jobInsert.Do()

	if jerr != nil {
		c.printDebug("Error inserting job!", jerr)
		if dataChan != nil {
			dataChan <- Data{Err: jerr}
		}
		return nil, nil, jerr
	}

	var qr *bigquery.GetQueryResultsResponse
	var rows [][]interface{}
	var headers []string
	var err error

	// Periodically, job references are not created, but errors are also not thrown.
	// In this scenario, retry up to 5 times to get a job reference before giving up.
	for i := 1; ; i++ {
		r := service.Jobs.GetQueryResults(project, runningJob.JobReference.JobId)
		r.TimeoutMs(c.RequestTimeout)
		qr, err = r.Do()

		headers, rows = c.headersAndRows(qr.Schema, qr.Rows)

		if i >= maxRequestRetry || qr.JobReference != nil || err != nil {
			if i > 1 {
				c.printDebug("Took %v tries to get a job reference", i)
			}
			break
		}
	}

	if err == nil && qr.JobReference == nil {
		err = fmt.Errorf("missing job reference")
	}

	if err != nil {
		c.printDebug("Error loading query: ", err)
		if dataChan != nil {
			dataChan <- Data{Err: err}
		}
		return nil, nil, err
	}

	rows, headers, err = c.processPagedQuery(qr.JobReference, qr.PageToken, dataChan, headers, rows)
	c.printDebug("largeDataPagedQuery completed in ", time.Now().Sub(ts).Seconds(), "s")

	return rows, headers, err
}
// largeDataPagedQuery builds a job and inserts it into the job queue allowing the flexibility to set the custom AllowLargeResults flag for the job