feat(Data Mapper): Cherry-picking January Data Mapper changes for public preview (#6470)

* Feat(Data Mapper): Panel with warning and error messages (#6315)

* started moving to v9 component

* api call works

* added value

* started test

* simplifying

* update to fix fluent ui bug

* component test passes

* fixed type error

* update version of fluent 9

* updated snapshots

* updated lockfile

* starting docs

* moved over v1 map checker

* simple panel works

* new error store

* added docs and displayed deserialization messages

* panels close when others open

* added required input checeker

* PR prep

* PR Comments

* updated yaml for tests

* feat(Data Mapper): Add keyboard delete, fix circles cutting, handle existing map loading (#6329)

* add keyboard delete

* only use delete when funciton or edge is selected

* remove backspace

* update width to handle circles cutting

* fix auto layouting

* Feat(Data Mapper): UX fixes for Public Preview (#6363)

* added new map placeholder

* added some colors

* custom info label

* PR prep

* Fix(Data Mapper): Sequence function serialization bug (#6378)

* fixed serialization issue

* test fix

* fix(Data Mapper): Fix circles cutting off (#6379)

data mapper update

* Fix(Data Mapper): bracket serialization bug (#6385)

* fixed bracket bug

* fixed typo in test

* feat(Data Mapper): Add notification to vscode from DM (#6390)

* setup vscode notification

* update toast to vscode

* fix(Data Mapper): Update Function config panel to fix wrapping, delete logic, placement of action items, info bubble  (#6393)

* dm font

* fix func config panel

* update function config 2

* fix style cherry-pick

* add remove logic

* fix margin

* feat(Data Mapper): Add info message and fix placeholders (#6405)

info message

* Fix(Data Mapper): add new files (#6396)

* starting using vscode file api

* fixing file fn

* file copy works and with schema imports

* error message works

* added info label

* fix(Data Mapper): Update height/width of panel and enhance Error/Warning panel (#6410)

* fix height

* enhance test panel

* fix(Data Mapper): Update string and open all functions by default (#6413)

open by default

* update strings

* merge main

* merge

* merge string changes

* update string

* Feat(Data Mapper): Loop message when users try to find loop in functions (#6451)

message works

* fix(Data Mapper): Fix unnecessary edge when node/parent is not in the search results (#6454)

* fix(Data Mapper): Pass in loading as a normal prop since boolean behaves differently (#6456)

loading passed

* fix(Data Mapper): Remove extra styling and clear out test results (#6459)

* update node position on search term change

* clear out test panel

* update circles

* fix(Data Mapper): Add subtitle for error/warning cards (#6464)

* node udpates

* add subtitle

---------

Co-authored-by: DanielleCogs <[email protected]>

Author: takyyon

Localize/lang/strings.json

@@ -50,6 +50,10 @@ class DataMapperFileService implements IDataMapperFileService {
     this.verbose = verbose;
   }
 
+  public sendNotification(title: string, text: string, level: number) {
+    console.log(`Notification: ${title}, data: ${text}, level: ${level}`);
+  }
+
   public saveMapDefinitionCall = (dataMapDefinition: string, mapMetadata: string) => {
     if (this.verbose) {
       console.log('Saved definition: ', dataMapDefinition);

apps/Standalone/src/dataMapperV1/app/DataMapperStandaloneDesignerV2.tsx

@@ -30,7 +30,7 @@ const mapDefinitionDropdownOptions: MapDefDropdownOption[] = [
     data: {
       mapDefinitionString: fullTranscriptMapDefinitionString,
       mapMetadataString: JSON.stringify(testMetadata),
-      associatedSchemaIdx: 2,
+      associatedSchemaIdx: 1,
     },
   },
   {
@@ -58,7 +58,6 @@ const sourceSchemaFileOptions: SchemaFileData[] = [
 ];
 const targetSchemaFileOptions: SchemaFileData[] = [
   { filename: 'PlaygroundTargetSchema.json', schemaFormat: SchemaFileFormat.XML },
-  { filename: 'OebsProjectRequest.json', schemaFormat: SchemaFileFormat.XML },
   { filename: 'TargetSchema.json', schemaFormat: SchemaFileFormat.XML },
   { filename: 'ComprehensiveTargetSchema.json', schemaFormat: SchemaFileFormat.XML },
   { filename: 'TargetSchemaJson.json', schemaFormat: SchemaFileFormat.JSON },

apps/Standalone/src/dataMapperV1/components/DevToolbox.tsx

@@ -0,0 +1,14 @@
+# Features
+Helper docs for how some of our features work
+## Error Panel
+Checks run by MapCheckerPanel
+- Component MapCheckerPanel updates and runs map checks using the connections and schema
+    - These are not currently stored in Redux, but could be if we allow users to dismiss warnings in the future
+- Currently we check for
+    -  Type checking for target schema nodes, but not functions until we get data from the backend
+    -  Missing required inputs on target nodes
+    - Missing required inputs for functions
+
+Deserialization Warnings
+- We generate warnings during deserialization if we are unable to find nodes or functions
+- These warnings are stored in Redux

apps/docs/docs/data-mapper/features.md

@@ -0,0 +1,15 @@
+
+# Serialization
+
+The process of converting the format used to represent the visual map canvas to LML that can be edited by the user and processed by the backend
+
+### GenerateMapDefinitionBody
+- filters connections for the ones that end in a target node
+- loops through these in random order and 
+    - generates key-value pair array for this connection by calling createNewPathItems; this is every pair in the LML from root to final target item
+    - calls applyValueAtPath for this array of pairs to insert it into the new LML, traversing down to where the pair belongs
+
+### createNewPathItems
+
+### applyValueAtPath
+- goes from the target connection passed, and using 'pathToRoot', adding all loops and conditionals to the path

apps/docs/docs/data-mapper/serialization.md

@@ -13,7 +13,7 @@ import {
   supportedSchemaFileExts,
   supportedCustomXsltFileExts,
 } from './extensionConfig';
-import type { SchemaType, MapMetadata, IFileSysTreeItem } from '@microsoft/logic-apps-shared';
+import { type SchemaType, type MapMetadata, type IFileSysTreeItem, LogEntryLevel } from '@microsoft/logic-apps-shared';
 import type { IActionContext } from '@microsoft/vscode-azext-utils';
 import { callWithTelemetryAndErrorHandlingSync } from '@microsoft/vscode-azext-utils';
 import type { MapDefinitionData, MessageToVsix, MessageToWebview } from '@microsoft/vscode-extension-logic-apps';
@@ -31,6 +31,7 @@ import * as path from 'path';
 import type { WebviewPanel } from 'vscode';
 import { RelativePattern, window, workspace } from 'vscode';
 import * as vscode from 'vscode';
+import { copyOverImportedSchemas } from './DataMapperPanelUtils';
 
 export default class DataMapperPanel {
   public panel: WebviewPanel;
@@ -114,7 +115,10 @@ export default class DataMapperPanel {
       }
       case ExtensionCommand.webviewLoaded: {
         // Send runtime port to webview
-        this.sendMsgToWebview({ command: ExtensionCommand.setRuntimePort, data: `${ext.designTimePort}` });
+        this.sendMsgToWebview({
+          command: ExtensionCommand.setRuntimePort,
+          data: `${ext.designTimePort}`,
+        });
 
         // If loading a data map, handle that + xslt filename
         this.handleLoadMapDefinitionIfAny();
@@ -127,7 +131,7 @@ export default class DataMapperPanel {
         break;
       }
       case ExtensionCommand.addSchemaFromFile: {
-        this.addSchemaFromFile(msg.data.path, msg.data.type);
+        this.addSchemaFromFile(msg.data);
         break;
       }
       case ExtensionCommand.readLocalSchemaFileOptions: {
@@ -174,6 +178,10 @@ export default class DataMapperPanel {
         ext.telemetryReporter.sendTelemetryEvent(eventName, { ...msg.data });
         break;
       }
+      case ExtensionCommand.sendNotification: {
+        this.sendNotification(msg.data.title, msg.data.text, msg.data.level);
+        break;
+      }
     }
   }
 
@@ -271,6 +279,28 @@ export default class DataMapperPanel {
     }
   }
 
+  public sendNotification(title: string, text: string, level: number) {
+    const msg = localize(title, text);
+    switch (level) {
+      case LogEntryLevel.Error: {
+        ext.showError(msg);
+        break;
+      }
+      case LogEntryLevel.Warning: {
+        ext.showWarning(msg);
+        break;
+      }
+      case LogEntryLevel.Verbose: {
+        ext.showInformation(msg);
+        break;
+      }
+      default: {
+        ext.log(msg);
+        break;
+      }
+    }
+  }
+
   private getFilesForPath(
     folderPath: string,
     command: typeof ExtensionCommand.showAvailableSchemas | typeof ExtensionCommand.getAvailableCustomXsltPaths,
@@ -301,51 +331,40 @@ export default class DataMapperPanel {
     });
   }
 
-  public addSchemaFromFile(filePath: string, schemaType: SchemaType) {
+  public addSchemaFromFile(schemaType: SchemaType) {
     callWithTelemetryAndErrorHandlingSync(extensionCommand.dataMapAddSchemaFromFile, (_context: IActionContext) => {
-      fs.readFile(filePath, 'utf8').then((text: string) => {
-        const primarySchemaFileName = path.basename(filePath); // Ex: inpSchema.xsd
-        const expectedPrimarySchemaPath = path.join(ext.logicAppWorkspace, schemasPath, primarySchemaFileName);
-
-        // Examine the loaded text for the 'schemaLocation' attribute to auto-load in any dependencies too
-        // NOTE: We only check in the same directory as the primary schema file (also, it doesn't attempt to deal with complicated paths/URLs, just filenames)
-        const schemaFileDependencies = [...text.matchAll(/schemaLocation="[A-Za-z.]*"/g)].map((schemaFileAttributeMatch) => {
-          // Trim down to just the filename
-          return schemaFileAttributeMatch[0].split('"')[1];
-        });
+      const fileSelectOptions: vscode.OpenDialogOptions = {
+        filters: { Schemas: ['xsd', 'json'] },
+        canSelectMany: false,
+      };
+      window.showOpenDialog(fileSelectOptions).then((files) => {
+        if (!files[0]) {
+          return;
+        }
+        const selectedFile = files[0];
 
-        schemaFileDependencies.forEach((schemaFile) => {
-          const schemaFilePath = path.join(path.dirname(filePath), schemaFile);
-
-          // Check that the schema file dependency exists in the same directory as the primary schema file
-          if (!fileExistsSync(schemaFilePath)) {
-            ext.showError(
-              localize(
-                'SchemaLoadingError',
-                `Schema loading error: couldn't find schema file dependency 
-              "{0}" in the same directory as "{1}". "{1}" will still be copied to the Schemas folder.`,
-                schemaFile,
-                primarySchemaFileName
-              )
-            );
-            return;
-          }
+        const pathToWorkspaceSchemaFolder = path.join(ext.logicAppWorkspace, schemasPath);
+        const primarySchemaFullPath = selectedFile.path;
+        const pathToContainingFolder = path.dirname(primarySchemaFullPath);
+        const primarySchemaFileName = path.basename(primarySchemaFullPath);
 
-          // Check that the schema file dependency doesn't already exist in the Schemas folder
-          const expectedSchemaFilePath = path.join(ext.logicAppWorkspace, schemasPath, schemaFile);
-          if (!fileExistsSync(expectedSchemaFilePath)) {
-            copyFileSync(schemaFilePath, expectedSchemaFilePath);
-          }
-        });
+        workspace.fs.readFile(selectedFile).then((fileContents) => {
+          const text = Buffer.from(fileContents).toString('utf-8');
 
-        // Check if in Artifacts/Schemas, and if not, create it and send it to DM for API call
-        if (!fileExistsSync(expectedPrimarySchemaPath)) {
-          copyFileSync(filePath, expectedPrimarySchemaPath);
-        }
+          copyOverImportedSchemas(text, primarySchemaFileName, pathToContainingFolder, pathToWorkspaceSchemaFolder, ext);
 
-        this.sendMsgToWebview({
-          command: ExtensionCommand.fetchSchema,
-          data: { fileName: primarySchemaFileName, type: schemaType as SchemaType },
+          const newPath = path.join(pathToWorkspaceSchemaFolder, primarySchemaFileName);
+          if (!fileExistsSync(newPath)) {
+            copyFileSync(primarySchemaFullPath, newPath);
+          }
+
+          this.sendMsgToWebview({
+            command: ExtensionCommand.fetchSchema,
+            data: {
+              fileName: primarySchemaFileName,
+              type: schemaType as SchemaType,
+            },
+          });
         });
       });
     });

apps/vs-code-designer/src/app/commands/dataMapper/DataMapperPanel.ts

@@ -0,0 +1,41 @@
+import path from 'path';
+import { copyFileSync, existsSync as fileExistsSync } from 'fs';
+import { localize } from '../../../localize';
+
+export const copyOverImportedSchemas = (
+  schemaText: string,
+  primarySchemaName: string,
+  pathToContainingFolder: string,
+  pathToWorkspaceSchemaFolder: string,
+  ext
+) => {
+  const schemaFileDependencies = [...schemaText.matchAll(/schemaLocation="[A-Za-z.]*"/g)].map((schemaFileAttributeMatch) => {
+    // Trim down to just the filename
+    return schemaFileAttributeMatch[0].split('"')[1];
+  });
+
+  schemaFileDependencies.forEach((importedSchemaFileName) => {
+    const importedSchemaFileFullPath = path.join(pathToContainingFolder, importedSchemaFileName);
+
+    // Check that the schema file dependency exists in the same directory as the primary schema file
+    if (!fileExistsSync(importedSchemaFileFullPath)) {
+      ext.showError(
+        localize(
+          'SchemaLoadingError',
+          `Schema loading error: couldn't find schema file import 
+                            "{0}" in the same directory as "{1}". "{2}" will still be copied to the Schemas folder.`,
+          importedSchemaFileName,
+          pathToContainingFolder,
+          primarySchemaName
+        )
+      );
+      return;
+    }
+
+    // Check that the schema file dependency doesn't already exist in the Schemas folder
+    const newSchemaFilePath = path.join(pathToWorkspaceSchemaFolder, importedSchemaFileName);
+    if (!fileExistsSync(newSchemaFilePath)) {
+      copyFileSync(importedSchemaFileFullPath, newSchemaFilePath);
+    }
+  });
+};

apps/vs-code-designer/src/app/commands/dataMapper/DataMapperPanelUtils.ts

@@ -3,6 +3,7 @@ export const webviewType = 'dataMapperWebview';
 export const supportedDataMapDefinitionFileExts = ['.lml', '.yml'];
 export const supportedSchemaFileExts = ['.xsd', '.json'];
 export const supportedCustomXsltFileExts = ['.xslt', '.xml'];
+export const supportedDataMapperFolders = ['Maps', 'MapDefinitions', 'Schemas'];
 
 const artifactsPath = '/Artifacts/';
 export const schemasPath = `${artifactsPath}/Schemas`;

apps/vs-code-designer/src/app/commands/dataMapper/extensionConfig.ts

@@ -166,6 +166,7 @@ export const extensionCommand = {
   dataMapSetSupportedDataMapDefinitionFileExts: 'azureLogicAppsStandard.dataMap.setSupportedDataMapDefinitionFileExts',
   dataMapSetSupportedSchemaFileExts: 'azureLogicAppsStandard.dataMap.setSupportedSchemaFileExts',
   dataMapSetSupportedFileExts: 'azureLogicAppsStandard.dataMap.setSupportedFileExts',
+  dataMapSetDmFolders: 'azureLogicAppsStandard.dataMap.setDmFolders',
   dataMapSaveMapDefinition: 'azureLogicAppsStandard.dataMap.saveMapDefinition',
   dataMapSaveMapXslt: 'azureLogicAppsStandard.dataMap.saveMapXslt',
   vscodeOpenFolder: 'vscode.openFolder',

apps/vs-code-designer/src/constants.ts

@@ -88,6 +88,10 @@ export namespace ext {
     window.showWarningMessage(errMsg);
   };
 
+  export const showInformation = (msg: string) => {
+    window.showInformationMessage(msg);
+  };
+
   export const showError = (errMsg: string, options?: MessageOptions) => {
     ext.log(errMsg);
     if (options && options.detail) {

apps/vs-code-designer/src/extensionVariables.ts

@@ -1,6 +1,10 @@
 import { LogicAppResolver } from './LogicAppResolver';
 import { runPostWorkflowCreateStepsFromCache } from './app/commands/createCodeless/createCodelessSteps/WorkflowCreateStepBase';
-import { supportedDataMapDefinitionFileExts, supportedSchemaFileExts } from './app/commands/dataMapper/extensionConfig';
+import {
+  supportedDataMapDefinitionFileExts,
+  supportedDataMapperFolders,
+  supportedSchemaFileExts,
+} from './app/commands/dataMapper/extensionConfig';
 import { promptParameterizeConnections } from './app/commands/parameterizeConnections';
 import { registerCommands } from './app/commands/registerCommands';
 import { getResourceGroupsApi } from './app/resourcesExtension/getExtensionApi';
@@ -36,6 +40,7 @@ const perfStats = {
 const telemetryString = 'setInGitHubBuild';
 
 export async function activate(context: vscode.ExtensionContext) {
+  // Data Mapper context
   vscode.commands.executeCommand(
     'setContext',
     extensionCommand.dataMapSetSupportedDataMapDefinitionFileExts,
@@ -46,6 +51,7 @@ export async function activate(context: vscode.ExtensionContext) {
     ...supportedDataMapDefinitionFileExts,
     ...supportedSchemaFileExts,
   ]);
+  vscode.commands.executeCommand('setContext', extensionCommand.dataMapSetDmFolders, supportedDataMapperFolders);
 
   ext.context = context;
   ext.telemetryReporter = new TelemetryReporter(telemetryString);

apps/vs-code-designer/src/main.ts

@@ -327,12 +327,14 @@
       },
       {
         "command": "azureLogicAppsStandard.dataMap.createNewDataMap",
-        "title": "Data Mapper: Create data map",
+        "title": "Create Data Map",
+        "category": "Azure Logic Apps",
         "icon": "$(add)"
       },
       {
         "command": "azureLogicAppsStandard.dataMap.loadDataMapFile",
-        "title": "Data Mapper: Load existing data map"
+        "title": "Data Mapper: Load existing data map",
+        "when": "resourceExtname in azureLogicAppsStandard.dataMap.setSupportedDataMapDefinitionFileExts"
       },
       {
         "command": "azureLogicAppsStandard.parameterizeConnections",
@@ -349,10 +351,6 @@
           "dark": "assets/logicapp.png"
         },
         "label": "Azure Logic Apps"
-      },
-      {
-        "id": "azureLogicAppsStandard.dataMapperMenu",
-        "label": "Data Mapper"
       }
     ],
     "views": {
@@ -627,6 +625,11 @@
         }
       ],
       "explorer/context": [
+        {
+          "command": "azureLogicAppsStandard.dataMap.createNewDataMap",
+          "when": "resourceFilename in azureLogicAppsStandard.dataMap.setDmFolders",
+          "group": "navigation"
+        },
         {
           "command": "azureLogicAppsStandard.deploy",
           "when": "explorerResourceIsFolder == true",
@@ -678,9 +681,9 @@
           "group": "navigation@2"
         },
         {
-          "submenu": "azureLogicAppsStandard.dataMapperMenu",
+          "command": "azureLogicAppsStandard.dataMap.loadDataMapFile",
           "group": "navigation",
-          "when": "resourceExtname in azureLogicAppsStandard.dataMap.setSupportedFileExts"
+          "when": "resourceExtname in azureLogicAppsStandard.dataMap.setSupportedDataMapDefinitionFileExts"
         }
       ],
       "commandPalette": [
@@ -716,13 +719,6 @@
           "command": "azureLogicAppsStandard.generateDeploymentScripts",
           "when": "never"
         }
-      ],
-      "azureLogicAppsStandard.dataMapperMenu": [
-        {
-          "command": "azureLogicAppsStandard.dataMap.loadDataMapFile",
-          "group": "navigation",
-          "when": "resourceExtname in azureLogicAppsStandard.dataMap.setSupportedDataMapDefinitionFileExts"
-        }
       ]
     },
     "configuration": [