Páginas

SyntaxHighlighter

sábado, 27 de julho de 2013

Dynamically generating Zip files using Google Cloud Storage Client Library for Appengine

Let's say you have 50 objects (15Mb each) stored in Google Cloud Storage. Now, you need to create a zip archive containing all of them and store the resultant file back into GCS. How can we achieve that from within an Appengine java application?
Well, after some research, I wrote the method below using Google Cloud Storage Client Library which does exactly that. Just don't forget  to grant the appropriate permissions to your appengine service account so that it can read and write the objects.


public static void zipFiles(final GcsFilename targetZipFile,
  final GcsFilename... filesToZip) throws IOException {

 Preconditions.checkArgument(targetZipFile != null);
 Preconditions.checkArgument(filesToZip != null);
 Preconditions.checkArgument(filesToZip.length > 0);

 final int fetchSize = 4 * 1024 * 1024;
 final int readSize = 2 * 1024 * 1024;
 GcsOutputChannel outputChannel = null;
 ZipOutputStream zip = null;
 try {
  final GcsFileOptions options = new GcsFileOptions.Builder()
    .mimeType(MediaType.ZIP.toString()).build();
  outputChannel = GCS_SERVICE.createOrReplace(targetZipFile, options);
  zip = new ZipOutputStream(Channels.newOutputStream(outputChannel));
  GcsInputChannel readChannel = null;
  for (final GcsFilename file : filesToZip) {
   try {
    final GcsFileMetadata meta = GCS_SERVICE.getMetadata(file);
    if (meta == null) {
     LOGGER.warning(file.toString()
       + " NOT FOUND. Skipping.");
     continue;
    }
    final ZipEntry entry = new ZipEntry(file.getObjectName());
    zip.putNextEntry(entry);
    readChannel = GCS_SERVICE.openPrefetchingReadChannel(file,
      0, fetchSize);
    final ByteBuffer buffer = ByteBuffer.allocate(readSize);
    int bytesRead = 0;
    while (bytesRead >= 0) {
     bytesRead = readChannel.read(buffer);
     buffer.flip();
     zip.write(buffer.array(), buffer.position(),
       buffer.limit());
     buffer.rewind();
     buffer.limit(buffer.capacity());
    }

   } finally {
    zip.closeEntry();
    readChannel.close();
   }
  }
 } finally {
  zip.flush();
  zip.close();
  outputChannel.close();
 }
}