Merge remote-tracking branch 'origin/bugfix/ARSN-402-batchDeleteReque…

…stLogger' into w/7.70/bugfix/ARSN-402-batchDeleteRequestLogger
scality · Mar 13, 2024 · a643a3e · a643a3e
2 parents 69dbbb1 + e9d815c
commit a643a3e
Show file tree

Hide file tree

Showing 21 changed files with 63 additions and 19 deletions.
diff --git a/lib/models/BackendInfo.ts b/lib/models/BackendInfo.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import { legacyLocations } from '../constants';
 import escapeForXml from '../s3middleware/escapeForXml';
 

diff --git a/lib/models/ReplicationConfiguration.ts b/lib/models/ReplicationConfiguration.ts
@@ -1,6 +1,8 @@
 import assert from 'assert';
 import UUID from 'uuid';
 
+import { RequestLogger } from 'werelogs';
+
 import escapeForXml from '../s3middleware/escapeForXml';
 import errors from '../errors';
 import { isValidBucketName } from '../s3routes/routesUtils';

diff --git a/lib/network/probe/ProbeServer.ts b/lib/network/probe/ProbeServer.ts
@@ -19,7 +19,7 @@ export const DEFAULT_METRICS_ROUTE = '/_/metrics';
  * string or undefined is used to represent no issues.
  */
 
-export type ProbeDelegate = (res: http.ServerResponse, log: RequestLogger) => string | void
+export type ProbeDelegate = (res: http.ServerResponse, log: werelogs.RequestLogger) => string | void
 
 export type ProbeServerParams = {
     port: number;

diff --git a/lib/network/rest/RESTClient.ts b/lib/network/rest/RESTClient.ts
@@ -116,7 +116,7 @@ export default class RESTClient {
         method: string,
         headers: http.OutgoingHttpHeaders | null,
         key: string | null,
-        log: RequestLogger,
+        log: werelogs.RequestLogger,
         responseCb: (res: http.IncomingMessage) => void,
     ) {
         const reqHeaders = headers || {};

diff --git a/lib/network/rest/RESTServer.ts b/lib/network/rest/RESTServer.ts
@@ -25,7 +25,7 @@ function setContentRange(
 
 function sendError(
     res: http.ServerResponse,
-    log: RequestLogger,
+    log: werelogs.RequestLogger,
     error: ArsenalError,
     optMessage?: string,
 ) {
@@ -177,7 +177,7 @@ export default class RESTServer extends httpServer {
     _onPut(
         req: http.IncomingMessage,
         res: http.ServerResponse,
-        log: RequestLogger,
+        log: werelogs.RequestLogger,
     ) {
         let size: number;
         try {
@@ -219,7 +219,7 @@ export default class RESTServer extends httpServer {
     _onGet(
         req: http.IncomingMessage,
         res: http.ServerResponse,
-        log: RequestLogger,
+        log: werelogs.RequestLogger,
     ) {
         let pathInfo: ReturnType<typeof parseURL>;
         let rangeSpec: ReturnType<typeof httpUtils.parseRangeSpec> | undefined =
@@ -302,7 +302,7 @@ export default class RESTServer extends httpServer {
     _onDelete(
         req: http.IncomingMessage,
         res: http.ServerResponse,
-        log: RequestLogger,
+        log: werelogs.RequestLogger,
     ) {
         let pathInfo: ReturnType<typeof parseURL>;
         try {

diff --git a/lib/s3middleware/azureHelpers/mpuUtils.ts b/lib/s3middleware/azureHelpers/mpuUtils.ts
@@ -1,6 +1,9 @@
 import assert from 'assert';
 import * as crypto from 'crypto';
 import * as stream from 'stream';
+
+import { RequestLogger } from 'werelogs';
+
 import ResultsCollector from './ResultsCollector';
 import SubStreamInterface from './SubStreamInterface';
 import * as objectUtils from '../objectUtils';

diff --git a/lib/s3routes/routes.ts b/lib/s3routes/routes.ts
@@ -1,4 +1,7 @@
 import assert from 'assert';
+
+import { RequestLogger } from 'werelogs';
+
 import errors from '../errors';
 import routeGET from './routes/routeGET';
 import routePUT from './routes/routePUT';

diff --git a/lib/s3routes/routes/routeDELETE.ts b/lib/s3routes/routes/routeDELETE.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import StatsClient from '../../metrics/StatsClient';

diff --git a/lib/s3routes/routes/routeGET.ts b/lib/s3routes/routes/routeGET.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';

diff --git a/lib/s3routes/routes/routeHEAD.ts b/lib/s3routes/routes/routeHEAD.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import StatsClient from '../../metrics/StatsClient';

diff --git a/lib/s3routes/routes/routeOPTIONS.ts b/lib/s3routes/routes/routeOPTIONS.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';

diff --git a/lib/s3routes/routes/routePOST.ts b/lib/s3routes/routes/routePOST.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';

diff --git a/lib/s3routes/routes/routePUT.ts b/lib/s3routes/routes/routePUT.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';

diff --git a/lib/s3routes/routes/routeWebsite.ts b/lib/s3routes/routes/routeWebsite.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import * as routesUtils from '../routesUtils';
 import errors from '../../errors';
 import * as http from 'http';

diff --git a/lib/s3routes/routesUtils.ts b/lib/s3routes/routesUtils.ts
@@ -1,10 +1,13 @@
 import * as url from 'url';
+import * as http from 'http';
+import { eachSeries } from 'async';
+
+import { RequestLogger } from 'werelogs';
+
 import * as ipCheck from '../ipCheck';
 import errors, { ArsenalError } from '../errors';
 import * as constants from '../constants';
-import { eachSeries } from 'async';
 import DataWrapper from '../storage/data/DataWrapper';
-import * as http from 'http';
 import StatsClient from '../metrics/StatsClient';
 
 export type CallApiMethod = (

diff --git a/lib/storage/data/DataWrapper.js b/lib/storage/data/DataWrapper.js
@@ -2,6 +2,8 @@ const async = require('async');
 const PassThrough = require('stream').PassThrough;
 const assert = require('assert');
 
+const { Logger } = require('werelogs');
+
 const errors = require('../../errors').default;
 const MD5Sum = require('../../s3middleware/MD5Sum').default;
 const NullStream = require('../../s3middleware/nullStream').default;
@@ -27,6 +29,7 @@ class DataWrapper {
         this.metadata = metadata;
         this.locStorageCheckFn = locStorageCheckFn;
         this.vault = vault;
+        this.logger = new Logger('DataWrapper');
     }
 
     put(cipherBundle, value, valueSize, keyContext, backendInfo, log, cb) {
@@ -127,7 +130,7 @@ class DataWrapper {
     }
 
     delete(objectGetInfo, log, cb) {
-        const callback = cb || log.end;
+        const callback = cb || (() => {});
         const isMdModelVersion2 = typeof(objectGetInfo) === 'string';
         const isRequiredStringKey =
             constants.clientsRequireStringKey[this.implName];
@@ -176,7 +179,9 @@ class DataWrapper {
             newObjDataStoreName)) {
             return process.nextTick(cb);
         }
-        log.trace('initiating batch delete', {
+        const delLog = this.logger.newRequestLoggerFromSerializedUids(
+            log.getSerializedUids());
+        delLog.trace('initiating batch delete', {
             keys: locations,
             implName: this.implName,
             method: 'batchDelete',
@@ -202,21 +207,21 @@ class DataWrapper {
             return false;
         });
         if (shouldBatchDelete && keys.length > 1) {
-            return this.client.batchDelete(backendName, { keys }, log, cb);
+            return this.client.batchDelete(backendName, { keys }, delLog, cb);
         }
         return async.eachLimit(locations, 5, (loc, next) => {
-            process.nextTick(() => this.delete(loc, log, next));
+            process.nextTick(() => this.delete(loc, delLog, next));
         },
         err => {
             if (err) {
-                log.end().error('batch delete failed', { error: err });
+                delLog.end().error('batch delete failed', { error: err });
                 // deletion of non-existing objects result in 204
                 if (err.code === 404) {
                     return cb();
                 }
                 return cb(err);
             }
-            log.end().trace('batch delete successfully completed');
+            delLog.end().trace('batch delete successfully completed');
             return cb();
         });
     }

diff --git a/lib/versioning/VersioningRequestProcessor.ts b/lib/versioning/VersioningRequestProcessor.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import errors, { ArsenalError } from '../errors';
 import { Version } from './Version';
 import { generateVersionId as genVID, getInfVid } from './VersionID';

diff --git a/lib/versioning/WriteCache.ts b/lib/versioning/WriteCache.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import errors, { ArsenalError } from '../errors';
 import WriteGatheringManager from './WriteGatheringManager';
 

diff --git a/lib/versioning/WriteGatheringManager.ts b/lib/versioning/WriteGatheringManager.ts
@@ -1,3 +1,5 @@
+import { RequestLogger } from 'werelogs';
+
 import { ArsenalError } from '../errors';
 
 const WG_TIMEOUT = 5; // batching period in milliseconds

diff --git a/package.json b/package.json
@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=16"
   },
-  "version": "7.70.28",
+  "version": "7.70.29",
   "description": "Common utilities for the S3 project components",
   "main": "build/index.js",
   "repository": {
@@ -49,7 +49,7 @@
     "sproxydclient": "github:scality/sproxydclient#8.0.4",
     "utf8": "2.1.2",
     "uuid": "^3.0.1",
-    "werelogs": "scality/werelogs#8.1.1",
+    "werelogs": "scality/werelogs#8.1.4",
     "xml2js": "~0.4.23"
   },
   "optionalDependencies": {

diff --git a/yarn.lock b/yarn.lock
@@ -2934,6 +2934,11 @@ fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6:
   resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
   integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=
 
+fast-safe-stringify@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884"
+  integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==
+
 fb-watchman@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/fb-watchman/-/fb-watchman-2.0.1.tgz#fc84fb39d2709cf3ff6d743706157bb5708a8a85"
@@ -6490,10 +6495,11 @@ werelogs@scality/werelogs#8.1.0:
   dependencies:
     safe-json-stringify "1.0.3"
 
-werelogs@scality/werelogs#8.1.1:
-  version "8.1.1"
-  resolved "https://codeload.github.com/scality/werelogs/tar.gz/240be22c47f1cae30df0c3c965cd701e6300bd18"
+werelogs@scality/werelogs#8.1.4:
+  version "8.1.4"
+  resolved "https://codeload.github.com/scality/werelogs/tar.gz/d6bec11df034c88a12959791eb7dd60913eb5f47"
   dependencies:
+    fast-safe-stringify "^2.1.1"
     safe-json-stringify "^1.2.0"
 
 werelogs@scality/werelogs#GA7.2.0.5: