all repos — s3brm @ c543e4f5a809b6216e18a72f1b34faed6c7c6ca6

S3 Backup Retention Manager

feat: 🎉 Setup basic script
Tim Izzo tim@octree.ch
Thu, 18 Aug 2022 12:16:13 +0200
commit

c543e4f5a809b6216e18a72f1b34faed6c7c6ca6

5 files changed, 157 insertions(+), 0 deletions(-)

jump to
A .gitignore

@@ -0,0 +1,1 @@

+env.sh
A date.ts

@@ -0,0 +1,19 @@

+export const getStartOfDay = (date: Date): Date => + new Date(new Date(date).setHours(0, 0, 0, 0)); + +export const getStartOfWeek = (date: Date): Date => { + const day = date.getDay(); + const diff = date.getDate() - day + (day == 0 ? -6 : 1); + const startOfWeek = new Date(new Date(date).setDate(diff)); + return getStartOfDay(startOfWeek); +}; + +export const getStartOfMonth = (date: Date): Date => + getStartOfDay(new Date(date.getFullYear(), date.getMonth(), 1)); + +export const isSameDate = (dateA: Date, dateB: Date): boolean => { + if (dateA.getDate() !== dateB.getDate()) return false; + else if (dateA.getMonth() !== dateB.getMonth()) return false; + else if (dateA.getFullYear() !== dateB.getFullYear()) return false; + else return true; +};
A env.sh.example

@@ -0,0 +1,10 @@

+export S3_ENDPOINT_URL= +export S3_ACCESS_KEY= +export S3_SECRET_KEY= +export S3_REGION= +export S3_BUCKET= + +export FILES_PREFIX= +export DAILY_BACKUP_INTERVAL=13 +export WEEKLY_BACKUP_INTERVAL=31 +export MONTHLY_BACKUP_INTERVAL=183
A mod.ts

@@ -0,0 +1,81 @@

+import { S3, S3Object } from "https://deno.land/x/s3@0.5.0/mod.ts"; +import { terminal } from "https://raw.githubusercontent.com/5ika/ink/master/mod.ts"; +import { getStartOfMonth, getStartOfWeek, isSameDate } from "./date.ts"; + +const debugMode = Deno.args.includes("--debug") || Deno.args.includes("-d"); + +if (debugMode) + terminal.log("<yellow>DEBUG MODE - Nothing will be deleted</yellow>"); + +const s3 = new S3({ + accessKeyID: Deno.env.get("S3_ACCESS_KEY")!, + secretKey: Deno.env.get("S3_SECRET_KEY")!, + region: Deno.env.get("S3_REGION")!, + endpointURL: Deno.env.get("S3_ENDPOINT_URL"), +}); + +const bucket = s3.getBucket(Deno.env.get("S3_BUCKET")!); +const items: AsyncGenerator<S3Object> = bucket.listAllObjects({ + prefix: Deno.env.get("FILES_PREFIX") ?? "", + batchSize: 1000, +}); + +const today: Date = new Date(); +const dailyBackupInterval = Deno.env.get("DAILY_BACKUP_INTERVAL") || 13; +const dailyBackupMaxDate: Date = new Date( + new Date().setDate(today.getDate() - parseInt(`${dailyBackupInterval}`)) +); +const weeklyBackupInterval = Deno.env.get("WEEKLY_BACKUP_INTERVAL") || 31; +const weeklyBackupMaxDate: Date = new Date( + new Date().setDate(today.getDate() - parseInt(`${weeklyBackupInterval}`)) +); +const monthlyBackupInterval = Deno.env.get("MONTHLY_BACKUP_INTERVAL") || 183; +const monthlyBackupMaxDate: Date = new Date( + new Date().setDate(today.getDate() - parseInt(`${monthlyBackupInterval}`)) +); + +for await (const item of items) { + // If item is a directory or has no date + if (item.size === 0 || !item?.lastModified || !item.key) continue; + + // Keep items wich is first day of month and has date < monthlyBackupInterval days ago + const startOfMonth = getStartOfMonth(item.lastModified); + if ( + item.lastModified > monthlyBackupMaxDate && + isSameDate(item.lastModified, startOfMonth) + ) { + terminal.log( + `<magenta>Keep ${item.key} [monthly backup, < ${monthlyBackupInterval} days]</magenta>` + ); + continue; + } + + // Keep items wich is first day of week and has date < 1 weeklyBackupInterval days ago + const startOfWeek = getStartOfWeek(item.lastModified); + if ( + item.lastModified > weeklyBackupMaxDate && + isSameDate(item.lastModified, startOfWeek) + ) { + terminal.log( + `<cyan>Keep ${item.key} [weekly backup, < ${weeklyBackupInterval} days]</cyan>` + ); + continue; + } + + // Keep items with date < dailyBackupInterval days ago + if (item.lastModified > dailyBackupMaxDate) { + terminal.log( + `<dim>Keep ${item.key} [daily backup, < ${dailyBackupInterval} days]</dim>` + ); + continue; + } + + // Delete others + terminal.log(`<red>Delete ${item.key}</red>`); + if (!debugMode) + try { + await bucket.deleteObject(item.key); + } catch (error) { + terminal.log(item.key, error); + } +}
A test.ts

@@ -0,0 +1,46 @@

+import { assertEquals } from "https://deno.land/std@0.152.0/testing/asserts.ts"; +import { + getStartOfDay, + getStartOfWeek, + getStartOfMonth, + isSameDate, +} from "./date.ts"; + +const date = new Date("2022-08-13"); + +Deno.test("getSartOfDay", () => { + const startOfDay: Date = getStartOfDay(date); + assertEquals(startOfDay.getHours(), 0); + assertEquals(startOfDay.getMinutes(), 0); + assertEquals(startOfDay.getMilliseconds(), 0); +}); + +Deno.test("getStartOfWeek returns start of week", () => { + const startOfWeek = getStartOfWeek(date); + assertEquals(startOfWeek.getDate(), 8); + assertEquals(startOfWeek.getMonth(), 7); // 7 = August + assertEquals(startOfWeek.getHours(), 0); + assertEquals(startOfWeek.getMinutes(), 0); + assertEquals(startOfWeek.getMilliseconds(), 0); +}); + +Deno.test("getStartOfMonth returns start of month", () => { + const startOfMonth = getStartOfMonth(date); + assertEquals(startOfMonth.getDate(), 1); + assertEquals(startOfMonth.getMonth(), 7); // 7 = August + assertEquals(startOfMonth.getHours(), 0); + assertEquals(startOfMonth.getMinutes(), 0); + assertEquals(startOfMonth.getMilliseconds(), 0); +}); + +Deno.test("isSameDate returns true for same dates but different hours", () => { + const dateB = new Date(new Date("2022-08-13").setHours(23, 10, 33)); + const comparaison = isSameDate(date, dateB); + assertEquals(comparaison, true); +}); + +Deno.test("isSameDate returns false for different dates", () => { + const dateB = new Date("2022-08-02"); + const comparaison = isSameDate(date, dateB); + assertEquals(comparaison, false); +});