fix(ingest) tableau use luid filter

metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py

@@ -1565,6 +1565,7 @@
         query_filter: dict = {},
     ) -> Iterable[dict]:
         query_filter = optimize_query_filter(query_filter)
+        logger.debug(f"Query filter {query_filter}")
 
         # Calls the get_connection_object_page function to get the objects,
         # and automatically handles pagination.
@@ -1574,6 +1575,7 @@
         self.report.num_filter_queries_by_connection_type[connection_type] += len(
             filter_pages
         )
+        logger.debug(f"Filter pages {filter_pages}")
 
         for filter_page in filter_pages:
             has_next_page = 1
@@ -1609,31 +1611,24 @@
             project_names: List[str] = [
                 project.name for project in self.tableau_project_registry.values()
             ]
-            projects = {c.PROJECT_NAME_WITH_IN: project_names}
+            logger.debug(f"Workbook emit: Project names: {project_names}")
 
             for workbook in self.get_connection_objects(
                 query=workbook_graphql_query,
                 connection_type=c.WORKBOOKS_CONNECTION,
-                query_filter=projects,
                 page_size=self.config.effective_workbook_page_size,
             ):
-                # This check is needed as we are using projectNameWithin which return project as per project name so if
-                # user want to ingest only nested project C from A->B->C then tableau might return more than one Project
-                # if multiple project has name C. Ideal solution is to use projectLuidWithin to avoid duplicate project,
-                # however Tableau supports projectLuidWithin in Tableau Cloud June 2022 / Server 2022.3 and later.
-                project_luid: Optional[str] = self._get_workbook_project_luid(workbook)
-                if project_luid not in self.tableau_project_registry.keys():
-                    wrk_name: Optional[str] = workbook.get(c.NAME)
-                    wrk_id: Optional[str] = workbook.get(c.ID)
-                    prj_name: Optional[str] = workbook.get(c.PROJECT_NAME)
-
-                    self.report.warning(
-                        title="Skipping Missing Workbook",
-                        message="Skipping workbook as its project is not present in project registry",
-                        context=f"workbook={wrk_name}({wrk_id}), project={prj_name}({project_luid})",
+                logger.debug(
+                    f"Evaluating project: {workbook.get(c.PROJECT_NAME)} workbook {workbook.get(c.NAME)}"
+                )
+                if workbook.get(c.PROJECT_NAME) not in project_names:
+                    logger.debug(
+                        f"Skipping workbook {workbook.get(c.NAME)} as project {workbook.get(c.PROJECT_NAME)} "
+                        f"is not in the project registry"
                     )
                     continue
 
+                logger.debug(f"Adding workbook {workbook.get(c.NAME)}")
                 yield from self.emit_workbook_as_container(workbook)
 
                 for sheet in workbook.get(c.SHEETS, []):

metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py

@@ -946,12 +946,15 @@
     after: Optional[str],
     qry_filter: str = "",
 ) -> dict:
+    if qry_filter:
+        qry_filter = f", filter: {{ {qry_filter} }}"
+        logger.debug(f"qry_filter: {qry_filter}")
     query = f"""
         query GetItems(
           $first: Int,
           $after: String
         ) {{
-            {connection_name} (  first: $first, after: $after, filter:{{ {qry_filter} }})
+            {connection_name} (  first: $first, after: $after{qry_filter})
             {{
                 nodes {main_query}
                 pageInfo {{

metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_constant.py

@@ -9,6 +9,7 @@
 ERRORS = "errors"
 NODES = "nodes"
 PROJECT_NAME_WITH_IN = "projectNameWithin"
+PROJECT_LUID_WITH_IN = "projectLuidWithin"
 WORKBOOKS_CONNECTION = "workbooksConnection"
 PROJECT_LUID = "projectLuid"
 NAME = "name"