Merging in commit 5f78b4a

in order to address Issue bazelbuild#14

Author: hutchk

rules_python/whl.py

@@ -19,6 +19,17 @@
 import re
 import zipfile
 
+# Putting this a package's __init__.py causes it to set __path__ so that it is
+# possible to import modules and subpackages from other directories on sys.path.
+INITPY_CONTENTS = '''
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)
+'''
+
 
 class Wheel(object):
 
@@ -62,7 +73,7 @@ def metadata(self):
         with whl.open(os.path.join(self._dist_info(), 'metadata.json')) as f:
           return json.loads(f.read().decode("utf-8"))
       except KeyError:
-          pass
+        pass
       # fall back to METADATA file (https://www.python.org/dev/peps/pep-0427/)
       with whl.open(os.path.join(self._dist_info(), 'METADATA')) as f:
         return self._parse_metadata(f.read().decode("utf-8"))
@@ -102,29 +113,66 @@ def extras(self):
 
   def expand(self, directory):
     with zipfile.ZipFile(self.path(), 'r') as whl:
+      names = set(whl.namelist())
       whl.extractall(directory)
 
+    # Workaround for https://github.com/bazelbuild/rules_python/issues/14
+    for initpy in self.get_init_paths(names):
+      with open(os.path.join(directory, initpy), 'w') as f:
+        f.write(INITPY_CONTENTS)
+
+  def get_init_paths(self, names):
+    # Overwrite __init__.py in these directories.
+    # (required as googleapis-common-protos has an empty __init__.py, which
+    # blocks google.api.core from google-cloud-core)
+    NAMESPACES = ["google/api"]
+
+    # Find package directories without __init__.py, or where the __init__.py
+    # must be overwritten to create a working namespace. This is based on
+    # Bazel's PythonUtils.getInitPyFiles().
+    init_paths = set()
+    for n in names:
+      if os.path.splitext(n)[1] not in ['.so', '.py', '.pyc']:
+        continue
+      while os.path.sep in n:
+        n = os.path.dirname(n)
+        initpy = os.path.join(n, '__init__.py')
+        initpyc = os.path.join(n, '__init__.pyc')
+        if (initpy in names or initpyc in names) and n not in NAMESPACES:
+          continue
+        init_paths.add(initpy)
+
+    return init_paths
+
   # _parse_metadata parses METADATA files according to https://www.python.org/dev/peps/pep-0314/
   def _parse_metadata(self, content):
     # TODO: handle fields other than just name
     name_pattern = re.compile('Name: (.*)')
-    return { 'name': name_pattern.search(content).group(1) }
+    return {'name': name_pattern.search(content).group(1)}
 
 
 parser = argparse.ArgumentParser(
     description='Unpack a WHL file as a py_library.')
 
-parser.add_argument('--whl', action='store',
-                    help=('The .whl file we are expanding.'))
+parser.add_argument(
+    '--whl', action='store', help=('The .whl file we are expanding.'))
+
+parser.add_argument(
+    '--requirements',
+    action='store',
+    help='The pip_import from which to draw dependencies.')
 
-parser.add_argument('--requirements', action='store',
-                    help='The pip_import from which to draw dependencies.')
+parser.add_argument(
+    '--directory',
+    action='store',
+    default='.',
+    help='The directory into which to expand things.')
 
-parser.add_argument('--directory', action='store', default='.',
-                    help='The directory into which to expand things.')
+parser.add_argument(
+    '--extras',
+    action='append',
+    help='The set of extras for which to generate library targets.')
 
-parser.add_argument('--extras', action='append',
-                    help='The set of extras for which to generate library targets.')
 
 def main():
   args = parser.parse_args()
@@ -149,24 +197,22 @@ def main():
     deps = [{dependencies}],
 )
 {extras}""".format(
-  requirements=args.requirements,
-  dependencies=','.join([
-    'requirement("%s")' % d
-    for d in whl.dependencies()
-  ]),
-  extras='\n\n'.join([
-    """py_library(
+        requirements=args.requirements,
+        dependencies=','.join(
+            ['requirement("%s")' % d for d in whl.dependencies()]),
+        extras='\n\n'.join([
+            """py_library(
     name = "{extra}",
     deps = [
         ":pkg",{deps}
     ],
 )""".format(extra=extra,
-            deps=','.join([
-                'requirement("%s")' % dep
-                for dep in whl.dependencies(extra)
-            ]))
-    for extra in args.extras or []
-  ])))
+            deps=','.join(
+                ['requirement("%s")' % dep
+                 for dep in whl.dependencies(extra)]))
+            for extra in args.extras or []
+        ])))
+
 
 if __name__ == '__main__':
   main()