Merge pull request bazelbuild#18 from mattmoor/just-par

This adds support for "extras".

Author: mattmoor

README.md

@@ -106,6 +106,10 @@ for dependencies, however, it is recommended that folks stick with the
 `requirement` pattern in case the need arises for us to make changes to this
 format in the future.
 
+["Extras"](
+https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras)
+will have a target of the extra name (in place of `pkg` above).
+
 ## Updating `docs/`
 
 All of the content (except `BUILD`) under `docs/` is generated.  To update the

WORKSPACE

@@ -26,8 +26,8 @@ sass_repositories()
 
 git_repository(
     name = "io_bazel_skydoc",
-    remote = "https://github.com/bazelbuild/skydoc.git",
     commit = "e9be81cf5be41e4200749f5d8aa2db7955f8aacc",
+    remote = "https://github.com/bazelbuild/skydoc.git",
 )
 
 load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories")
@@ -92,6 +92,15 @@ http_file(
            "mock-2.0.0-py2.py3-none-any.whl"),
 )
 
+http_file(
+    name = "google_cloud_language_whl",
+    sha256 = "a2dd34f0a0ebf5705dcbe34bd41199b1d0a55c4597d38ed045bd183361a561e9",
+    # From https://pypi.python.org/pypi/google-cloud-language
+    url = ("https://pypi.python.org/packages/6e/86/" +
+           "cae57e4802e72d9e626ee5828ed5a646cf4016b473a4a022f1038dba3460/" +
+           "google_cloud_language-0.29.0-py2.py3-none-any.whl"),
+)
+
 # Imports for examples
 pip_import(
     name = "examples_helloworld",
@@ -128,3 +137,15 @@ load(
 )
 
 _boto_install()
+
+pip_import(
+    name = "examples_extras",
+    requirements = "//examples/extras:requirements.txt",
+)
+
+load(
+    "@examples_extras//:requirements.bzl",
+    _extras_install = "pip_install",
+)
+
+_extras_install()

examples/extras/BUILD

@@ -0,0 +1,29 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+load("@examples_extras//:requirements.bzl", "requirement")
+load("//python:python.bzl", "py_test")
+
+py_test(
+    name = "extras_test",
+    srcs = ["extras_test.py"],
+    deps = [
+        requirement("google-cloud-language"),
+        # Make sure that we can resolve the "extra" dependency
+        requirement("googleapis-common-protos[grpc]"),
+    ],
+)

examples/extras/extras_test.py

@@ -0,0 +1,25 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+
+# The test is the build itself, which should not work if extras are missing.
+class ExtrasTest(unittest.TestCase):
+  def test_nothing(self):
+    pass
+
+
+if __name__ == '__main__':
+  unittest.main()

examples/extras/requirements.txt

@@ -0,0 +1 @@
+google-cloud-language==0.27.0

examples/extras/version_test.py

@@ -0,0 +1,26 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pip
+import unittest
+
+
+class VersionTest(unittest.TestCase):
+
+  def test_version(self):
+    self.assertEqual(pip.__version__, '9.0.1')
+
+
+if __name__ == '__main__':
+  unittest.main()

python/whl.bzl

@@ -16,12 +16,17 @@
 def _whl_impl(repository_ctx):
   """Core implementation of whl_library."""
 
-  result = repository_ctx.execute([
+  args = [
     "python",
     repository_ctx.path(repository_ctx.attr._script),
     "--whl", repository_ctx.path(repository_ctx.attr.whl),
     "--requirements", repository_ctx.attr.requirements,
-  ])
+  ]
+
+  if repository_ctx.attr.extras:
+    args += ["--extras", ",".join(repository_ctx.attr.extras)]
+
+  result = repository_ctx.execute(args)
   if result.return_code:
     fail("whl_library failed: %s (%s)" % (result.stdout, result.stderr))
 
@@ -33,6 +38,7 @@ whl_library = repository_rule(
             single_file = True,
         ),
         "requirements": attr.string(),
+        "extras": attr.string_list(),
         "_script": attr.label(
             executable = True,
             default = Label("//rules_python:whl.py"),
@@ -64,4 +70,7 @@ Args:
 
   requirements: The name of the pip_import repository rule from which to
     load this .whl's dependencies.
+
+  extras: A subset of the "extras" available from this <code>.whl</code> for which
+    <code>requirements</code> has the dependencies.
 """

rules_python/BUILD

@@ -28,6 +28,7 @@ py_test(
     data = [
         "@futures_3_1_1_whl//file",
         "@futures_2_2_0_whl//file",
+        "@google_cloud_language_whl//file",
         "@grpc_whl//file",
         "@mock_whl//file",
     ],

rules_python/piptool.py

@@ -18,6 +18,7 @@
 import json
 import os
 import pkgutil
+import pkg_resources
 import re
 import shutil
 import sys
@@ -78,36 +79,7 @@ def pip_main(argv):
     argv = ["--disable-pip-version-check", "--cert", cert_path] + argv
     return pip.main(argv)
 
-
-# TODO(mattmoor): We can't easily depend on other libraries when
-# being invoked as a raw .py file.  Once bundled, we should be able
-# to remove this fallback on a stub implementation of Wheel.
-try:
-  from rules_python.whl import Wheel
-except:
-  class Wheel(object):
-
-    def __init__(self, path):
-      self._path = path
-
-    def basename(self):
-      return os.path.basename(self._path)
-
-    def distribution(self):
-      # See https://www.python.org/dev/peps/pep-0427/#file-name-convention
-      parts = self.basename().split('-')
-      return parts[0]
-
-    def version(self):
-      # See https://www.python.org/dev/peps/pep-0427/#file-name-convention
-      parts = self.basename().split('-')
-      return parts[1]
-
-    def repository_name(self):
-      # Returns the canonical name of the Bazel repository for this package.
-      canonical = 'pypi__{}_{}'.format(self.distribution(), self.version())
-      # Escape any illegal characters with underscore.
-      return re.sub('[-.]', '_', canonical)
+from rules_python.whl import Wheel
 
 parser = argparse.ArgumentParser(
     description='Import Python dependencies into Bazel.')
@@ -124,6 +96,59 @@ def repository_name(self):
 parser.add_argument('--directory', action='store',
                     help=('The directory into which to put .whl files.'))
 
+def determine_possible_extras(whls):
+  """Determines the list of possible "extras" for each .whl
+
+  The possibility of an extra is determined by looking at its
+  additional requirements, and determinine whether they are
+  satisfied by the complete list of available wheels.
+
+  Args:
+    whls: a list of Wheel objects
+
+  Returns:
+    a dict that is keyed by the Wheel objects in whls, and whose
+    values are lists of possible extras.
+  """
+  whl_map = {
+    whl.distribution(): whl
+    for whl in whls
+  }
+
+  # TODO(mattmoor): Consider memoizing if this recursion ever becomes
+  # expensive enough to warrant it.
+  def is_possible(distro, extra):
+    distro = distro.replace("-", "_")
+    # If we don't have the .whl at all, then this isn't possible.
+    if distro not in whl_map:
+      return False
+    whl = whl_map[distro]
+    # If we have the .whl, and we don't need anything extra then
+    # we can satisfy this dependency.
+    if not extra:
+      return True
+    # If we do need something extra, then check the extra's
+    # dependencies to make sure they are fully satisfied.
+    for extra_dep in whl.dependencies(extra=extra):
+      req = pkg_resources.Requirement.parse(extra_dep)
+      # Check that the dep and any extras are all possible.
+      if not is_possible(req.project_name, None):
+        return False
+      for e in req.extras:
+        if not is_possible(req.project_name, e):
+          return False
+    # If all of the dependencies of the extra are satisfiable then
+    # it is possible to construct this dependency.
+    return True
+
+  return {
+    whl: [
+      extra
+      for extra in whl.extras()
+      if is_possible(whl.distribution(), extra)
+    ]
+    for whl in whls
+  }
 
 def main():
   args = parser.parse_args()
@@ -140,6 +165,9 @@ def list_whls():
         if fname.endswith('.whl'):
           yield os.path.join(root, fname)
 
+  whls = [Wheel(path) for path in list_whls()]
+  possible_extras = determine_possible_extras(whls)
+
   def whl_library(wheel):
     # Indentation here matters.  whl_library must be within the scope
     # of the function below.  We also avoid reimporting an existing WHL.
@@ -149,10 +177,25 @@ def whl_library(wheel):
         name = "{repo_name}",
         whl = "@{name}//:{path}",
         requirements = "@{name}//:requirements.bzl",
+        extras = [{extras}]
     )""".format(name=args.name, repo_name=wheel.repository_name(),
-              path=wheel.basename())
-
-  whls = [Wheel(path) for path in list_whls()]
+                path=wheel.basename(),
+                extras=','.join([
+                  '"%s"' % extra
+                  for extra in possible_extras.get(wheel, [])
+                ]))
+
+  whl_targets = ','.join([
+    ','.join([
+      '"%s": "@%s//:pkg"' % (whl.distribution().lower(), whl.repository_name())
+    ] + [
+      # For every extra that is possible from this requirements.txt
+      '"%s[%s]": "@%s//:%s"' % (whl.distribution().lower(), extra.lower(),
+                                whl.repository_name(), extra)
+      for extra in possible_extras.get(whl, [])
+    ])
+    for whl in whls
+  ])
 
   with open(args.output, 'w') as f:
     f.write("""\
@@ -178,10 +221,7 @@ def requirement(name):
   return _requirements[name_key]
 """.format(input=args.input,
            whl_libraries='\n'.join(map(whl_library, whls)) if whls else "pass",
-           mappings=','.join([
-             '"%s": "@%s//:pkg"' % (wheel.distribution().lower(), wheel.repository_name())
-             for wheel in whls
-           ])))
+           mappings=whl_targets))
 
 if __name__ == '__main__':
   main()