CODEWITHSUNDEEP
javagoogle-bigquery
WOPR
WOPR

Reputation: 5393

Google BigQuery Socket Timeout

I'm downloading, via the Java API, a large query result set from BigQuery.

After about 1 hour, the underlying socket is closed, causing a SocketException error and the query fails.

How can I extend the socket timeout, or preferably, disable it? Here's how I'm building a connection.

  InputStream credentialsJSON = new FileInputStream("path.json");


  NetHttpTransport transport = new NetHttpTransport();
  JsonFactory jsonFactory = new JacksonFactory();
  GoogleCredential credential = GoogleCredential.fromStream(credentialsJSON, transport, jsonFactory);
  credential.setExpiresInSeconds(null);

  // Inject the BigQuery scope if required.
  if (credential.createScopedRequired()) {
      Collection<String> bigqueryScopes = BigqueryScopes.all();
      credential = credential.createScoped(bigqueryScopes);
  }

  return new Bigquery.Builder(transport, jsonFactory, credential)
          .setApplicationName("ETL")
          .build();

}

Stack trace

java.net.SocketTimeoutException: Read timed out
    at java.net.SocketInputStream.socketRead0(Native Method)
    at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
    at java.net.SocketInputStream.read(SocketInputStream.java:170)
    at java.net.SocketInputStream.read(SocketInputStream.java:141)
    at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
    at sun.security.ssl.InputRecord.read(InputRecord.java:503)
    at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:961)
    at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:918)
    at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
    at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
    at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
    at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
    at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:704)
    at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:647)
    at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1535)
    at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1440)
    at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480)
    at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:338)
    at com.google.api.client.http.javanet.NetHttpResponse.<init>(NetHttpResponse.java:37)
    at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:94)
    at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:972)
    at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419)
    at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352)
    at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:469)
    at Common.BigQueryUtils$1PageIterator.next(BigQueryUtils.java:204)
    at Common.BigQueryUtils$1PageIterator.next(BigQueryUtils.java:173)
    at ATTRWarehouse.BigQueryPuller.retrieve(BigQueryPuller.java:270)
    at ATTRWarehouse.BigQueryPuller.pullData(BigQueryPuller.java:121)
    at Common.BaseDataPuller.call(BaseDataPuller.java:123)

Upvotes: 0

Views: 1834

Answers (1)

WOPR
WOPR

Reputation: 5393

OK I solved this by extending the HTTP timeout by overriding its initialiser.

  private static HttpRequestInitializer setHttpTimeout(final HttpRequestInitializer requestInitializer) {
    return new HttpRequestInitializer() {
        @Override
        public void initialize(HttpRequest httpRequest) throws IOException {
           requestInitializer.initialize(httpRequest);
           httpRequest.setConnectTimeout(5 * 60000);  // 5 minutes connect timeout
           httpRequest.setReadTimeout(360 * 60000);  // 120 minutes read timeout
        }
        };
   }





    private static Bigquery createAuthorizedClient() throws IOException {

        InputStream credentialsJSON = new FileInputStream("src/main/java/Warehouse/Common/GoogleCredential.json");


        NetHttpTransport transport = new NetHttpTransport();
        JsonFactory jsonFactory = new JacksonFactory();
        GoogleCredential credential = GoogleCredential.fromStream(credentialsJSON, transport, jsonFactory);

        // Inject the BigQuery scope if required.
        if (credential.createScopedRequired()) {
            Collection<String> bigqueryScopes = BigqueryScopes.all();
            credential = credential.createScoped(bigqueryScopes);
        }

        return new Bigquery.Builder(transport, jsonFactory, setHttpTimeout(credential))
                .setApplicationName("ETL")
                .build();
    }

Upvotes: 2

Related Questions