refactor: create a separate database for dbt

Moves dbt models into a separate database in ClickHouse for more
granular permissions

Author: SoryRawyer

README.rst

@@ -9,6 +9,7 @@ Aspects Learner Analytics combines several free, open source, tools to add analy
 - `Vector <https://vector.dev/>`__, a log forwarding tool that can be used to forward tracking log and xAPI data to ClickHouse
 - `event-routing-backends <https://https://event-routing-backends.readthedocs.io/en/latest/>`__, an Open edX plugin that transforms tracking logs into xAPI and optionally forwards them to one or more Learning Record Stores in near real time
 - `event-sink-clickhouse <https://github.com/openedx/openedx-event-sink-clickhouse>`__, an Open edX plugin that exports course structure and high level data to ClickHouse at publish time
+- `dbt <https://www.getdbt.com/>`__, a tool to build data pipelines from SQL queries. The dbt project used by this plugin is `aspects-dbt <https://github.com/openedx/aspects-dbt>`__.
 
 See https://github.com/openedx/openedx-oars for more details about the Aspects architecture and high level documentation.
 

tutoraspects/plugin.py

@@ -186,6 +186,8 @@
         # This is a pip compliant list of Python packages to install to run dbt
         # make sure packages with versions are enclosed in double quotes
         ("EXTRA_DBT_PACKAGES", []),
+        # This is the name of the database dbt will write to
+        ("DBT_PROFILE_TARGET_DATABASE", "reporting"),
         # If set, DDL/table operations will be executed with the `ON CLUSTER` clause
         # using this cluster. This has not been tested with Aspects and is unlikely to
         # work.

tutoraspects/templates/aspects/apps/aspects/dbt/profiles.yml

@@ -3,7 +3,7 @@ aspects: # this needs to match the profile in your dbt_project.yml file
   outputs:
     dev:
       type: clickhouse
-      schema: {{ ASPECTS_XAPI_DATABASE }}
+      schema: {{ DBT_PROFILE_TARGET_DATABASE }}
       host: {{ CLICKHOUSE_HOST }}
       port: {{ CLICKHOUSE_PORT }}
       user: {{ CLICKHOUSE_ADMIN_USER }}

tutoraspects/templates/aspects/apps/superset/pythonpath/assets.yaml

@@ -692,7 +692,7 @@
     warning_text: null
   offset: 0
   params: null
-  schema: {{ ASPECTS_XAPI_DATABASE }}
+  schema: {{ DBT_PROFILE_TARGET_DATABASE }}
   sql: ''
   table_name: video_plays
   template_params: null
@@ -789,7 +789,7 @@
     warning_text: null
   offset: 0
   params: null
-  schema: {{ ASPECTS_XAPI_DATABASE }}
+  schema: {{ DBT_PROFILE_TARGET_DATABASE }}
   sql: ''
   table_name: transcript_usage
   template_params: null

tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh

@@ -335,7 +335,7 @@ CREATE TABLE IF NOT EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SI
 
 -- Grant permissions to the users
 GRANT INSERT, SELECT ON {{ ASPECTS_XAPI_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_LRS_USER }}';
-GRANT SELECT, DROP TABLE, DROP VIEW ON {{ ASPECTS_XAPI_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}';
+GRANT SELECT ON {{ DBT_PROFILE_TARGET_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}';
 
 GRANT INSERT, SELECT ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_CMS_USER }}';
 GRANT SELECT ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}';