Cleanup client logs

Author: krzysztofzuraw

apps/avatax/package.json

@@ -7,14 +7,14 @@
     "build": "next build",
     "check-types": "tsc --noEmit",
     "cleanup-client-logs": "pnpm tsx --require dotenv/config ./scripts/cleanup-client-logs.ts",
+    "cleanup-client-logs:dry-run": "pnpm tsx --require dotenv/config ./scripts/cleanup-client-logs.ts --dry-run",
     "deploy": "tsx --require dotenv/config ./scripts/deploy.ts",
     "dev": "NODE_OPTIONS='--inspect' next dev",
     "dev:debug": "APP_LOG_LEVEL=debug pnpm dev",
     "e2e": "vitest --project e2e --watch=false",
     "e2e:watch": "vitest --project e2e --watch",
     "fetch-schema": "curl https://raw.githubusercontent.com/saleor/saleor/${npm_package_saleor_schemaVersion}/saleor/graphql/schema.graphql > graphql/schema.graphql",
     "generate": "pnpm run generate:app && pnpm run generate:e2e",
-    "generate-fake-logs": "pnpm tsx --require dotenv/config ./scripts/generate-fake-logs.ts",
     "generate:app": "graphql-codegen --config .graphqlrc.ts",
     "generate:e2e": "graphql-codegen --config .graphqlrc.ts --project e2e",
     "lint": "eslint .",
@@ -57,7 +57,7 @@
     "avatax": "^23.7.0",
     "decimal.js-light": "2.5.1",
     "dotenv": "16.3.1",
-    "dynamodb-toolbox": "1.16.3",
+    "dynamodb-toolbox": "1.8.2",
     "graphql": "16.7.1",
     "graphql-tag": "2.12.6",
     "jotai": "^2.4.2",

apps/avatax/scripts/cleanup-client-logs.ts

@@ -1,6 +1,6 @@
 import { parseArgs } from "node:util";
 
-import { DeletePartitionCommand } from "dynamodb-toolbox/table/actions/deletePartition";
+import { DeleteItemCommand, QueryCommand } from "dynamodb-toolbox";
 import { saleorApp } from "saleor-app";
 
 import { env } from "@/env";
@@ -16,10 +16,12 @@ const { values } = parseArgs({
   },
 });
 
-const startDate = new Date(2024, 1, 1);
-const endDate = new Date(2025, 2, 24); // 14 days from today
+const today = new Date();
+const endDate = new Date(today.getTime() - 14 * 24 * 60 * 60 * 1000); // 14 days from today
 
 const main = async () => {
+  const start = performance.now();
+
   const dynamoClient = createLogsDynamoClient();
 
   const logsTable = LogsTable.create({
@@ -38,28 +40,62 @@ const main = async () => {
   });
 
   for (const { saleorApiUrl, appId } of appInstallations) {
-    const command = logsTable
-      .build(DeletePartitionCommand)
-      .entities(logsByCheckoutOrOrderId, logsByDateEntity)
-      .query({
-        partition: LogsTable.getPrimaryKey({ saleorApiUrl, appId }),
-        range: {
-          between: [startDate.toISOString(), endDate.toISOString()],
-        },
-      });
-
     try {
+      let lastEvaluatedKey: Record<string, unknown> | undefined = undefined;
+
+      const command = logsTable
+        .build(QueryCommand)
+        .query({
+          partition: LogsTable.getPrimaryKey({ saleorApiUrl, appId }),
+          lte: endDate,
+        })
+        .entities(logsByCheckoutOrOrderId, logsByDateEntity);
+
       if (values["dry-run"]) {
         console.log(`Would delete logs for ${saleorApiUrl}#${appId}`);
         continue;
       }
+
       console.log(`Deleting logs for ${saleorApiUrl}#${appId}`);
-      await command.send();
+
+      do {
+        const page = await command
+          .options({
+            limit: 100,
+            exclusiveStartKey: lastEvaluatedKey,
+          })
+          .send();
+
+        for (const item of page?.Items ?? []) {
+          if (item.checkoutOrOrderId) {
+            await logsByCheckoutOrOrderId
+              .build(DeleteItemCommand)
+              .key({
+                PK: item.PK,
+                SK: item.SK,
+                checkoutOrOrderId: item.checkoutOrOrderId,
+                date: item.date,
+              })
+              .send();
+          } else {
+            await logsByDateEntity
+              .build(DeleteItemCommand)
+              .key({
+                PK: item.PK,
+                SK: item.SK,
+                date: item.date,
+              })
+              .send();
+          }
+        }
+        lastEvaluatedKey = page.LastEvaluatedKey;
+      } while (lastEvaluatedKey !== undefined);
       console.log(`Logs for ${saleorApiUrl}#${appId} deleted`);
     } catch (error) {
       console.error(`Could not delete logs for ${saleorApiUrl}#${appId}`, error);
     }
   }
+  console.log(`Cleanup took ${performance.now() - start}ms`);
 };
 
 main();