Updated web data uploads to be turned off by default

Author: aschonfeld

.circleci/config.yml

@@ -155,6 +155,7 @@ python: &python
             . ci/bin/activate
             # xlrd & xarray do not load correctly from "python setup.py develop"
             # possibly switch this script to use "pip install -r requirements.txt"
+            pip install pillow
             pip install xlrd
             pip install xarray
             if [ "${CIRCLE_JOB}" == "build_3_10" ]; then
@@ -191,6 +192,9 @@ python: &python
           command: |
             set -e
             . ci/bin/activate
+            if [ "${CIRCLE_JOB}" == "build_2_7" ]; then
+              pip install backports.functools-lru-cache==1.6.6
+            fi
             if [ "${CIRCLE_JOB}" != "build_2_7" ]; then
               pip install -e ".[arcticdb]"
             fi

README.md

@@ -1352,6 +1352,19 @@ Here's the options at you disposal:
   * pandas.util.testing.makeTimeDataFrame
 
 
+**Starting with version 3.8.1 web uploads will be turned off by default.
+Web uploads are vulnerable to blind server side request forgery, please only use in trusted environments.**
+
+**You can turn this feature on by doing one of the following:**
+ - **add `enable_web_uploads=True` to your `dtale.show` call**
+ - **add `enable_web_uploads = False` to the [app] section of your dtale.ini config file ([more info](https://github.com/man-group/dtale/blob/master/docs/CONFIGURATION.md))**
+ - **run this code before calling dtale.show:**
+```python
+import dtale.global_state as global_state
+global_state.set_app_settings(dict(enable_web_uploads=True))
+```
+
+
 #### Instances
 This will give you information about other D-Tale instances are running under your current Python process.
 

docs/CONFIGURATION.md

@@ -26,6 +26,7 @@ hide_header_menu = False
 hide_main_menu = False
 hide_column_menus = False
 enable_custom_filters = False
+enable_web_uploads = False
 
 [charts] # this controls how many points can be contained within scatter & 3D charts
 scatter_points = 15000

dtale/__init__.py

@@ -23,6 +23,7 @@
     HIDE_MAIN_MENU = False
     HIDE_COLUMN_MENUS = False
     ENABLE_CUSTOM_FILTERS = False
+    ENABLE_WEB_UPLOADS = False
 
     # flake8: NOQA
     from dtale.app import show, get_instance, instances, offline_chart  # isort:skip

dtale/app.py

@@ -721,6 +721,8 @@ def show(data=None, data_loader=None, name=None, context_vars=None, **options):
     :type highlight_filter: boolean, optional
     :param enable_custom_filters: If true, this will enable users to make custom filters from the UI
     :type enable_custom_filters: bool, optional
+    :param enable_web_uploads: If true, this will enable users to upload files using URLs from the UI
+    :type enable_web_uploads: bool, optional
 
     :Example:
 
@@ -807,6 +809,7 @@ def show(data=None, data_loader=None, name=None, context_vars=None, **options):
             hide_main_menu=final_options.get("hide_main_menu"),
             hide_column_menus=final_options.get("hide_column_menus"),
             enable_custom_filters=final_options.get("enable_custom_filters"),
+            enable_web_uploads=final_options.get("enable_web_uploads"),
         )
         instance.started_with_open_browser = final_options["open_browser"]
         is_active = not running_with_flask_debug() and is_up(app_url)

dtale/config.py

@@ -114,6 +114,13 @@ def load_app_settings(config):
         section="app",
         getter="getboolean",
     )
+    enable_web_uploads = get_config_val(
+        config,
+        curr_app_settings,
+        "enable_web_uploads",
+        section="app",
+        getter="getboolean",
+    )
     open_custom_filter_on_startup = get_config_val(
         config,
         curr_app_settings,
@@ -153,6 +160,7 @@ def load_app_settings(config):
             hide_main_menu=hide_main_menu,
             hide_column_menus=hide_column_menus,
             enable_custom_filters=enable_custom_filters,
+            enable_web_uploads=enable_web_uploads,
         )
     )
 
@@ -223,6 +231,7 @@ def build_show_options(options=None):
         hide_main_menu=None,
         hide_column_menus=None,
         enable_custom_filters=None,
+        enable_web_uploads=None,
     )
     config_options = {}
     config = get_config()

dtale/datasets.py

@@ -1,6 +1,9 @@
+import numpy as np
 import pandas as pd
 import requests
+import string
 import zipfile
+from datetime import datetime
 
 from six import BytesIO
 
@@ -89,11 +92,19 @@ def movies():
 
 
 def time_dataframe():
-    try:
-        from pandas._testing import makeTimeDataFrame
-
-        return makeTimeDataFrame(), None
-    except ImportError:
-        from pandas.util.testing import makeTimeDataFrame
-
-        return makeTimeDataFrame(), None
+    def series_data():
+        if hasattr(np.random, "default_rng"):
+            return np.random.default_rng(2).standard_normal(30)
+        return np.random.randn(30)
+
+    cols = string.ascii_uppercase[:4]
+    data = {
+        c: pd.Series(
+            series_data(),
+            index=pd.DatetimeIndex(
+                pd.date_range(datetime(2000, 1, 1), periods=30, freq="B")
+            ),
+        )
+        for c in cols
+    }
+    return pd.DataFrame(data), None

dtale/global_state.py

@@ -33,6 +33,7 @@
     "hide_main_menu": False,
     "hide_column_menus": False,
     "enable_custom_filters": False,
+    "enable_web_uploads": False,
 }
 
 AUTH_SETTINGS = {"active": False, "username": None, "password": None}
@@ -616,6 +617,15 @@ def set_app_settings(settings):
                     "use in trusted environments."
                 )
             )
+    if settings.get("enable_web_uploads") is not None:
+        instance_updates["enable_web_uploads"] = settings.get("enable_web_uploads")
+        if instance_updates["enable_web_uploads"]:
+            logger.warning(
+                (
+                    "Turning on Web uploads. Web uploads are vulnerable to blind server side request forgery, please "
+                    "only use in trusted environments."
+                )
+            )
 
     if _default_store.size() > 0 and len(instance_updates):
         for data_id in _default_store.keys():

dtale/templates/dtale/base.html

@@ -45,6 +45,7 @@
         <input type="hidden" id="hide_main_menu" value="{{hide_main_menu}}" />
         <input type="hidden" id="hide_column_menus" value="{{hide_column_menus}}" />
         <input type="hidden" id="enable_custom_filters" value="{{enable_custom_filters}}" />
+        <input type="hidden" id="enable_web_uploads" value="{{enable_web_uploads}}" />
         <input type="hidden" id="allow_cell_edits" value="{{allow_cell_edits}}" />
         <input type="hidden" id="hide_drop_rows" value="{{hide_drop_rows}}" />
         <input type="hidden" id="is_vscode" value="{{is_vscode}}" />

dtale/views.py

@@ -352,6 +352,7 @@ def update_settings(self, **updates):
         * hide_main_menu - if true, this will hide the main menu from the screen
         * hide_column_menus - if true, this will hide the column menus from the screen
         * enable_custom_filters - if True, allow users to specify custom filters from the UI using pandas.query strings
+        * enable_web_uploads - if True, allow users to upload files using URLs from the UI
 
         After applying please refresh any open browsers!
         """
@@ -916,6 +917,7 @@ def startup(
     hide_main_menu=None,
     hide_column_menus=None,
     enable_custom_filters=None,
+    enable_web_uploads=None,
     force_save=True,
 ):
     """
@@ -1044,6 +1046,7 @@ def startup(
             hide_main_menu=hide_main_menu,
             hide_column_menus=hide_column_menus,
             enable_custom_filters=enable_custom_filters,
+            enable_web_uploads=enable_web_uploads,
         )
         startup_code = (
             "from arcticdb import Arctic\n"
@@ -1116,6 +1119,7 @@ def startup(
                 hide_main_menu=hide_main_menu,
                 hide_column_menus=hide_column_menus,
                 enable_custom_filters=enable_custom_filters,
+                enable_web_uploads=enable_web_uploads,
             )
 
             global_state.set_dataset(instance._data_id, data)
@@ -1185,6 +1189,8 @@ def startup(
             base_settings["hide_column_menus"] = hide_column_menus
         if enable_custom_filters is not None:
             base_settings["enable_custom_filters"] = enable_custom_filters
+        if enable_web_uploads is not None:
+            base_settings["enable_web_uploads"] = enable_web_uploads
         if column_edit_options is not None:
             base_settings["column_edit_options"] = column_edit_options
         global_state.set_settings(data_id, base_settings)
@@ -1239,6 +1245,13 @@ def startup(
                     "use in trusted environments."
                 )
             )
+        if global_state.load_flag(data_id, "enable_web_uploads", False):
+            logger.warning(
+                (
+                    "Web uploads enabled. Web uploads are vulnerable to blind server side request forgery, please "
+                    "only use in trusted environments."
+                )
+            )
         return DtaleData(data_id, url, is_proxy=is_proxy, app_root=app_root)
     else:
         raise NoDataLoadedException("No data has been loaded into this D-Tale session!")
@@ -1275,6 +1288,7 @@ def base_render_template(template, data_id, **kwargs):
     enable_custom_filters = global_state.load_flag(
         data_id, "enable_custom_filters", False
     )
+    enable_web_uploads = global_state.load_flag(data_id, "enable_web_uploads", False)
     app_overrides = dict(
         allow_cell_edits=json.dumps(allow_cell_edits),
         hide_shutdown=hide_shutdown,
@@ -1284,6 +1298,7 @@ def base_render_template(template, data_id, **kwargs):
         hide_main_menu=hide_main_menu,
         hide_column_menus=hide_column_menus,
         enable_custom_filters=enable_custom_filters,
+        enable_web_uploads=enable_web_uploads,
         github_fork=github_fork,
     )
     is_arcticdb = 0
@@ -3926,6 +3941,17 @@ def web_upload():
     from dtale.cli.loaders.excel_loader import load_file as load_excel
     from dtale.cli.loaders.parquet_loader import loader_func as load_parquet
 
+    if not global_state.get_app_settings().get("enable_web_uploads", False):
+        return jsonify(
+            dict(
+                success=False,
+                error=(
+                    "Web uploads not enabled! Web uploads are vulnerable to blind server side request forgery, please "
+                    "only use in trusted environments."
+                ),
+            )
+        )
+
     data_type = get_str_arg(request, "type")
     url = get_str_arg(request, "url")
     proxy = get_str_arg(request, "proxy")

frontend/static/__tests__/dtale/upload/Upload.test.support.tsx

@@ -1,11 +1,14 @@
 import { act, render } from '@testing-library/react';
 import axios from 'axios';
 import * as React from 'react';
+import { Provider } from 'react-redux';
+import { Store } from 'redux';
 
 import Upload from '../../../popups/upload/Upload';
 import { Dataset, DataType } from '../../../popups/upload/UploadState';
 import * as UploadRepository from '../../../repository/UploadRepository';
 import reduxUtils from '../../redux-test-utils';
+import { buildInnerHTML } from '../../test-utils';
 
 /** Bundles alot of jest setup for CreateColumn component tests */
 export class Spies {
@@ -17,6 +20,7 @@ export class Spies {
   public presetUploadSpy: jest.SpyInstance<Promise<UploadRepository.UploadResponse | undefined>, [dataset: Dataset]>;
   public readAsDataURLSpy: jest.SpyInstance;
   public btoaSpy: jest.SpyInstance;
+  public store: Store;
 
   /** Initializes all spy instances */
   constructor() {
@@ -25,6 +29,7 @@ export class Spies {
     this.presetUploadSpy = jest.spyOn(UploadRepository, 'presetUpload');
     this.readAsDataURLSpy = jest.spyOn(FileReader.prototype, 'readAsDataURL');
     this.btoaSpy = jest.spyOn(window, 'btoa');
+    this.store = reduxUtils.createDtaleStore();
   }
 
   /** Sets the mockImplementation/mockReturnValue for spy instances */
@@ -48,11 +53,21 @@ export class Spies {
   /**
    * Build the initial wrapper.
    *
+   * @param overrides redux overrides
    * @return the wrapper for testing.
    */
-  public async setupWrapper(): Promise<Element> {
+  public async setupWrapper(overrides?: Record<string, string>): Promise<Element> {
+    this.store = reduxUtils.createDtaleStore();
+    buildInnerHTML({ enableWebUploads: 'True', ...overrides }, this.store);
     return await act((): Element => {
-      const { container } = render(<Upload />);
+      const { container } = render(
+        <Provider store={this.store}>
+          <Upload />
+        </Provider>,
+        {
+          container: document.getElementById('content') ?? undefined,
+        },
+      );
       return container;
     });
   }

frontend/static/__tests__/dtale/upload/upload-url-test.tsx

@@ -0,0 +1,48 @@
+import { screen } from '@testing-library/react';
+
+import { DISABLED_URL_UPLOADS_MSG } from '../../../popups/upload/Upload';
+
+import * as TestSupport from './Upload.test.support';
+
+describe('Upload', () => {
+  const { close, location, open, opener } = window;
+  const spies = new TestSupport.Spies();
+
+  beforeEach(async () => {
+    delete (window as any).location;
+    delete (window as any).close;
+    delete (window as any).open;
+    delete window.opener;
+    (window as any).location = {
+      reload: jest.fn(),
+      pathname: '/dtale/column/1',
+      href: '',
+      assign: jest.fn(),
+    };
+    window.close = jest.fn();
+    window.open = jest.fn();
+    window.opener = { location: { assign: jest.fn(), pathname: '/dtale/column/1' } };
+    spies.setupMockImplementations();
+    await spies.setupWrapper({ enableWebUploads: 'False' });
+  });
+
+  afterEach(() => spies.afterEach());
+
+  afterAll(() => {
+    spies.afterAll();
+    window.location = location;
+    window.close = close;
+    window.open = open;
+    window.opener = opener;
+  });
+
+  const upload = (): HTMLElement => screen.getByTestId('upload');
+
+  it('renders successfully', async () => {
+    expect(upload()).toBeDefined();
+  });
+
+  it('DataViewer: disabled web uploads', async () => {
+    expect(upload().getElementsByClassName('form-group')[0].textContent).toBe(DISABLED_URL_UPLOADS_MSG);
+  });
+});

frontend/static/__tests__/reducers/dtale-test.tsx

@@ -29,6 +29,7 @@ describe('reducer tests', () => {
       hideMainMenu: false,
       hideColumnMenus: false,
       enableCustomFilters: false,
+      enableWebUploads: false,
       hideDropRows: false,
       iframe: false,
       columnMenuOpen: false,

frontend/static/__tests__/test-utils.tsx

@@ -96,6 +96,7 @@ export const buildInnerHTML = (props: Record<string, string | undefined> = {}, s
     buildHidden('hide_main_menu', props.hideMainMenu ?? HIDE_SHUTDOWN),
     buildHidden('hide_column_menus', props.hideColumnMenus ?? HIDE_SHUTDOWN),
     buildHidden('enable_custom_filters', props.enableCustomFilters ?? HIDE_SHUTDOWN),
+    buildHidden('enable_web_uploads', props.enableWebUploads ?? HIDE_SHUTDOWN),
     BASE_HTML,
   ].join('');
   store?.dispatch(actions.init());

frontend/static/popups/create/LabeledInput.tsx

@@ -5,7 +5,7 @@ interface DtaleInputProps {
   type?: React.HTMLInputTypeAttribute;
   value?: any;
   setter: (value: string) => void;
-  inputOptions?: Partial<React.HTMLAttributes<HTMLInputElement>>;
+  inputOptions?: Partial<React.AllHTMLAttributes<HTMLInputElement>>;
 }
 
 const DtaleInput: React.FC<DtaleInputProps> = ({ type = 'text', value, setter, inputOptions }) => (

frontend/static/popups/filter/FilterPopup.tsx

@@ -28,7 +28,7 @@ export const DISABLED_CUSTOM_FILTERS_MSG = [
   '- add "enable_custom_filters=True" to your dtale.show call\n',
   '- run this code before calling dtale.show\n',
   '\timport dtale.global_state as global_state\n\tglobal_state.set_app_settings(dict(enable_custom_filters=True))\n',
-  '- add "enable_custom_filters = False" to the [app] section of your dtale.ini config file',
+  '- add "enable_custom_filters = True" to the [app] section of your dtale.ini config file',
 ].join('');
 
 export const selectResult = createSelector(