Reputation: 735
I'm using the npm library archiver to backup a Google Cloud Storage bucket into another bucket as .zip
file. It looks that my function works well when the zip file generated is not very big. In contrast it fails without log error when the backup zip file created is bigger (around 3GB more or less). The process ends without more information. This is the my function:
async function makeBackup (sourceStorage: Storage, targetStorage: Storage, sourceBucketName: string): Promise<void> {
if (targetStorage instanceof GoogleCloudStorage && sourceStorage instanceof GoogleCloudStorage) {
const now = Temporal.Now.instant()
const filename = now.toString().concat(`-${sourceBucketName}`).concat('-backup.zip')
console.log(` ${pointer} creating backup from %s bucket to %s (%s)`, sourceBucketName, sourceBucketName, filename)
const targetFile = targetStorage.getBucket().file(filename)
const targetStream = targetStorage.getBucket().file(filename).createWriteStream()
const archive = archiver('zip', {zlib: {level: 1}}) // compress level
archive.pipe(targetStream)
let totalFiles = 0
let totalSize = 0
const query = {autoPaginate: false, maxResults: 1000, pageToken: undefined}
do {
const [files, _, info] = await sourceStorage.getBucket().getFiles(query)
for (const file of files) {
console.log(` ${pointer} adding file: %s - size: %s`, file.metadata.name, file.metadata.size)
const fileStream = file.createReadStream()
archive.append(fileStream, {name: file.name})
totalFiles += 1
totalSize += parseInt(file.metadata.size)
}
query.pageToken = info.nextPageToken
} while (query.pageToken != null)
await archive.finalize()
await new Promise((resolve) => targetStream.on('finish', resolve))
const targetMetadata = await targetFile.getMetadata()
console.log(chalkTpl`{green [OK]} succesfully backed up %s files - size: %s - zipped size: %s`, totalFiles, filesize(totalSize), filesize(targetMetadata[0].size))
} else {
console.log(chalkTpl`{red [FAIL]} origin or destination storages are not a GCS`)
}
}
Is there something wrong in my function? Is a archiver
bug?
Upvotes: 0
Views: 50