Creating a "datasets" module

datasets/__init__.py

@@ -0,0 +1,2 @@
+from . import src
+from .src import *

datasets/src/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ["beer_reviews", "colleges", "crime_data", "employee_salaries",
+           "journal_influence", "medical_charge", "met_objects",
+           "midwest_survey", "open_payments", "road_safety",
+           "traffic_violations"]

src/beer_reviews.py → datasets/src/beer_reviews.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 
-from common.file_management import fetch, write_df
+from .common.file_management import fetch, write_df
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/colleges.py → datasets/src/colleges.py

@@ -4,7 +4,7 @@
 import numpy as np
 import pandas as pd
 
-from common.file_management import fetch, write_df, float_to_int
+from .common.file_management import fetch, write_df, float_to_int
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/common/file_management.py → datasets/src/common/file_management.py

@@ -17,12 +17,11 @@ def _unzip(file, data_dir='./'):
     z.close()
 
 
-def _untargz(file, ext):
-    gz = gzip.open(file)
-    out = open(file, 'wb')
-    shutil.copyfileobj(gz, out, 8192)
-    gz.close()
-    out.close()
+def _untargz(file, data_dir):
+    out = os.path.join(data_dir, os.path.splitext(file)[0])
+    with gzip.open(file) as gz:
+        with open(out, 'wb') as f_out:
+            shutil.copyfileobj(gz, f_out, 8192)
 
 
 def _untar(file, data_dir):
@@ -42,7 +41,7 @@ def _uncompress_file(file, data_dir, delete_archive):
         _unzip(file, data_dir)
         processed = True
     elif ext == '.gz' or header.startswith(b'\x1f\x8b'):
-        _untargz(file, ext)
+        _untargz(file, data_dir)
         processed = True
     if os.path.isfile(file) and tarfile.is_tarfile(file):
         _untar(file, data_dir)

src/crime_data.py → datasets/src/crime_data.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 
-from common.file_management import fetch, write_df, float_to_int
+from .common.file_management import fetch, write_df, float_to_int
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/employee_salaries.py → datasets/src/employee_salaries.py

@@ -4,7 +4,7 @@
 
 import pandas as pd
 
-from common.file_management import fetch, write_df
+from .common.file_management import fetch, write_df
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/journal_influence.py → datasets/src/journal_influence.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 
-from common.file_management import fetch, write_df, float_to_int
+from .common.file_management import fetch, write_df, float_to_int
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/medical_charge.py → datasets/src/medical_charge.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 
-from common.file_management import fetch, write_df
+from .common.file_management import fetch, write_df
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/met_objects.py → datasets/src/met_objects.py

@@ -4,7 +4,7 @@
 import numpy as np
 import pandas as pd
 
-from common.file_management import fetch, write_df
+from .common.file_management import fetch, write_df
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/midwest_survey.py → datasets/src/midwest_survey.py

@@ -4,7 +4,7 @@
 import numpy as np
 import pandas as pd
 
-from common.file_management import fetch, write_df
+from .common.file_management import fetch, write_df
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/open_payments.py → datasets/src/open_payments.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 
-from common.file_management import fetch, write_df
+from .common.file_management import fetch, write_df
 
 amount = ['Total_Amount_of_Payment_USDollars']
 corp = ['Applicable_Manufacturer_or_Applicable_GPO_Making_' + 'Payment_Name']

src/road_safety.py → datasets/src/road_safety.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import xlrd
 
-from common.file_management import fetch, write_df, float_to_int
+from .common.file_management import fetch, write_df, float_to_int
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

src/test.py → datasets/src/test.py

@@ -1,14 +1,14 @@
-from colleges import get_colleges_df
-from crime_data import get_crime_df
-from employee_salaries import get_employee_salaries_df
-from journal_influence import get_journal_influence_df
-from medical_charge import get_medical_charge_df
-from met_objects import get_met_objects_df
-from midwest_survey import get_midwest_survey_df
-from open_payments import get_open_payment_df
-from road_safety import get_road_safety_df
-from traffic_violations import get_traffic_violations_df
-from beer_reviews import get_beer_reviews_df
+from datasets.src.colleges import get_colleges_df
+from datasets.src.crime_data import get_crime_df
+from datasets.src.employee_salaries import get_employee_salaries_df
+from datasets.src.journal_influence import get_journal_influence_df
+from datasets.src.medical_charge import get_medical_charge_df
+from datasets.src.met_objects import get_met_objects_df
+from datasets.src.midwest_survey import get_midwest_survey_df
+from datasets.src.open_payments import get_open_payment_df
+from datasets.src.road_safety import get_road_safety_df
+from datasets.src.traffic_violations import get_traffic_violations_df
+from datasets.src.beer_reviews import get_beer_reviews_df
 import asyncio
 
 func_map = {

src/traffic_violations.py → datasets/src/traffic_violations.py

@@ -4,7 +4,7 @@
 import numpy as np
 import pandas as pd
 
-from common.file_management import fetch, write_df, float_to_int
+from .common.file_management import fetch, write_df, float_to_int
 
 DatasetInfo = namedtuple('DatasetInfo', ['name', 'urlinfos', 'main_file', 'source'])
 UrlInfo = namedtuple('UrlInfo', ['url', 'filenames', 'uncompress'])

setup.py

@@ -0,0 +1,30 @@
+import os
+from setuptools import setup, find_packages
+
+description_file = os.path.join(os.path.dirname(__file__), 'README.md')
+with open(description_file) as fh:
+    DESCRIPTION = fh.read()
+
+
+if __name__ == '__main__':
+    setup(name='datasets',
+          version='0.0.1',
+          author='Patricio Cerda',
+          author_email='[email protected]',
+          description=("Datasets for prediction with string categorical variables"),
+          long_description=DESCRIPTION,
+          license='BSD',
+          classifiers=[
+              'Environment :: Console',
+              'Intended Audience :: Science/Research',
+              'License :: OSI Approved :: BSD License',
+              'Operating System :: OS Independent',
+              'Programming Language :: Python :: 3.5',
+              'Programming Language :: Python :: 3.6',
+              'Topic :: Scientific/Engineering',
+              'Topic :: Software Development :: Libraries',
+          ],
+          platforms='any',
+          packages=find_packages(),
+          install_requires=['pandas', 'requests'],
+          )