Skip to content

Commit 6b634ba

Browse files
SoryRawyerSoryRawyer
committed
feat: add location to block display names
1 parent 1d36391 commit 6b634ba

File tree

2 files changed

+216
-0
lines changed

2 files changed

+216
-0
lines changed

‎tutoraspects/plugin.py

+2
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@
7777
("ASPECTS_EVENT_SINK_OVERVIEWS_TABLE", "course_overviews"),
7878
("ASPECTS_EVENT_SINK_USER_PROFILE_TABLE", "user_profile"),
7979
("ASPECTS_EVENT_SINK_CLICKHOUSE_TIMEOUT_SECS", "5"),
80+
("ASPECTS_EVENT_SINK_RECENT_BLOCKS_TABLE", "most_recent_course_blocks"),
81+
("ASPECTS_EVENT_SINK_RECENT_BLOCKS_MV", "most_recent_course_blocks_mv"),
8082
# Vector settings
8183
("ASPECTS_DOCKER_HOST_SOCK_PATH", "/var/run/docker.sock"),
8284
("ASPECTS_VECTOR_DATABASE", "openedx"),

‎tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0023_extend_display_names.py

+214
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
from alembic import op
2+
3+
revision = "0023"
4+
down_revision = "0022"
5+
branch_labels = None
6+
depends_on = None
7+
on_cluster = (
8+
" ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' "
9+
if "{{CLICKHOUSE_CLUSTER_NAME}}"
10+
else ""
11+
)
12+
engine = (
13+
"ReplicatedReplacingMergeTree"
14+
if "{{CLICKHOUSE_CLUSTER_NAME}}"
15+
else "ReplacingMergeTree"
16+
)
17+
18+
19+
def drop_objects():
20+
op.execute(
21+
f"""
22+
DROP TABLE IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_RECENT_BLOCKS_TABLE }}
23+
{on_cluster}
24+
"""
25+
)
26+
27+
op.execute(
28+
f"""
29+
DROP VIEW IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_RECENT_BLOCKS_MV }}
30+
{on_cluster}
31+
"""
32+
)
33+
34+
# We include these drop statements here because "CREATE OR REPLACE DICTIONARY"
35+
# currently throws a file rename error and you can't drop a dictionary with a
36+
# table referring to it.
37+
op.execute(
38+
f"""
39+
DROP TABLE IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names
40+
{on_cluster}
41+
"""
42+
)
43+
op.execute(
44+
f"""
45+
DROP DICTIONARY IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names_dict
46+
{on_cluster}
47+
"""
48+
)
49+
50+
51+
def upgrade():
52+
drop_objects()
53+
54+
op.execute(
55+
f"""
56+
CREATE TABLE IF NOT EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_RECENT_BLOCKS_TABLE }}
57+
{on_cluster}
58+
(
59+
location String NOT NULL,
60+
display_name String NOT NULL,
61+
display_name_with_location String NOT NULL,
62+
section Int32,
63+
subsection Int32,
64+
unit Int32,
65+
graded Bool,
66+
course_key String,
67+
dump_id UUID NOT NULL,
68+
time_last_dumped String NOT NULL
69+
) engine = {engine}
70+
PRIMARY KEY (location);
71+
"""
72+
)
73+
74+
op.execute(
75+
f"""
76+
create materialized view if not exists {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_RECENT_BLOCKS_MV }}
77+
{on_cluster}
78+
to {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_RECENT_BLOCKS_TABLE }} as
79+
select
80+
location,
81+
display_name,
82+
toString(section) || ':' || toString(subsection) || ':' || toString(unit) || ' - ' || display_name as display_name_with_location,
83+
JSONExtractInt(xblock_data_json, 'section') as section,
84+
JSONExtractInt(xblock_data_json, 'subsection') as subsection,
85+
JSONExtractInt(xblock_data_json, 'unit') as unit,
86+
JSONExtractBool(xblock_data_json, 'graded') as graded,
87+
course_key,
88+
dump_id,
89+
time_last_dumped
90+
from {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}
91+
"""
92+
)
93+
94+
op.execute(
95+
"""
96+
insert into {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_RECENT_BLOCKS_TABLE }} (
97+
location, display_name, display_name_with_location, section, subsection, unit, graded, course_key, dump_id, time_last_dumped
98+
)
99+
select
100+
location,
101+
display_name,
102+
toString(section) || ':' || toString(subsection) || ':' || toString(unit) || '- ' || display_name as display_name_with_location,
103+
JSONExtractInt(xblock_data_json, 'section') as section,
104+
JSONExtractInt(xblock_data_json, 'subsection') as subsection,
105+
JSONExtractInt(xblock_data_json, 'unit') as unit,
106+
JSONExtractBool(xblock_data_json, 'graded') as graded,
107+
course_key,
108+
dump_id,
109+
time_last_dumped
110+
from {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }};
111+
"""
112+
)
113+
114+
op.execute(
115+
f"""
116+
CREATE DICTIONARY {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names_dict
117+
{on_cluster}
118+
(
119+
location String,
120+
block_name String,
121+
course_key String,
122+
graded Bool,
123+
display_name_with_location String
124+
)
125+
PRIMARY KEY location
126+
SOURCE(CLICKHOUSE(
127+
user '{{ CLICKHOUSE_ADMIN_USER }}'
128+
password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
129+
db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
130+
query "
131+
select
132+
location,
133+
display_name,
134+
course_key,
135+
graded,
136+
display_name_with_location
137+
from {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_RECENT_BLOCKS_TABLE }}
138+
final
139+
"
140+
))
141+
LAYOUT(COMPLEX_KEY_SPARSE_HASHED())
142+
LIFETIME(120);
143+
"""
144+
)
145+
146+
op.execute(
147+
f"""
148+
CREATE OR REPLACE TABLE {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names
149+
{on_cluster}
150+
(
151+
location String,
152+
block_name String,
153+
course_key String,
154+
graded Bool,
155+
display_name_with_location String
156+
) engine = Dictionary({{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names_dict)
157+
;
158+
"""
159+
)
160+
161+
162+
def downgrade():
163+
drop_objects()
164+
op.execute(
165+
f"""
166+
CREATE DICTIONARY {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names_dict
167+
{on_cluster}
168+
(
169+
location String,
170+
block_name String,
171+
course_key String,
172+
graded Bool
173+
)
174+
PRIMARY KEY location
175+
SOURCE(CLICKHOUSE(
176+
user '{{ CLICKHOUSE_ADMIN_USER }}'
177+
password '{{ CLICKHOUSE_ADMIN_PASSWORD }}'
178+
db '{{ ASPECTS_EVENT_SINK_DATABASE }}'
179+
query "with most_recent_blocks as (
180+
select org, course_key, location, max(edited_on) as last_modified
181+
from {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}
182+
group by org, course_key, location
183+
)
184+
select
185+
location,
186+
display_name,
187+
course_key,
188+
JSONExtractBool(xblock_data_json, 'graded') as graded
189+
from {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }} co
190+
inner join most_recent_blocks mrb on
191+
co.org = mrb.org and
192+
co.course_key = mrb.course_key and
193+
co.location = mrb.location and
194+
co.edited_on = mrb.last_modified
195+
"
196+
))
197+
LAYOUT(COMPLEX_KEY_SPARSE_HASHED())
198+
LIFETIME(120);
199+
"""
200+
)
201+
202+
op.execute(
203+
f"""
204+
CREATE OR REPLACE TABLE {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names
205+
{on_cluster}
206+
(
207+
location String,
208+
block_name String,
209+
course_key String,
210+
graded Bool
211+
) engine = Dictionary({{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names_dict)
212+
;
213+
"""
214+
)

0 commit comments

Comments
 (0)