From 4654a91e3839ea1c18e52c31f0ff8f90525b5df6 Mon Sep 17 00:00:00 2001 From: Paul Cody Johnston Date: Sun, 22 Oct 2017 20:43:02 -0600 Subject: [PATCH] Refactor node_binary rule. * Improved support for various execution contexts. * Move node_modules tree creation into own rule. * parse yarn modules: create a node_binary rules foreach executable target in the module. * Redo mocha_test as an sh_test. * Add the node_test rule. --- .travis.yml | 3 +- Makefile | 30 +- README.md | 72 ++- WORKSPACE | 1 + node/BUILD | 4 + node/internal/mocha_test.bzl | 144 ++--- node/internal/mocha_test.sh | 2 + node/internal/node_binary.bzl | 308 +++++----- node/internal/node_bundle.bzl | 37 ++ node/internal/node_module.bzl | 67 ++- node/internal/node_modules.bzl | 102 ++++ node/internal/node_repositories.bzl | 2 +- node/internal/parse_yarn_lock.js | 526 +++++++++++++++--- node/internal/yarn_modules.bzl | 42 +- node/rules.bzl | 4 + tests/helloworld/BUILD | 19 +- tests/helloworld/gen/BUILD | 44 ++ tests/helloworld/gen/compiler.js | 2 + .../gen/generated_helloworld_test.sh | 8 + tests/helloworld/gen/input.js | 1 + tests/helloworld/helloworld_test.sh | 2 +- tests/helloworld/require_test.js | 4 + tests/lyrics/BUILD | 3 +- tests/mocha/BUILD | 13 +- tests/mocha/src/main/js/BUILD | 9 + tests/mocha/src/main/js/greeter.js | 3 + tests/mocha/{tests => src/main/js}/test.js | 0 tests/mocha/test.js | 7 + tests/mocha/tests/BUILD | 16 - tests/typescript/BUILD | 9 - tests/typescript/ts_module.bzl | 138 +++-- tests/webpack/BUILD | 40 ++ tests/webpack/README.md | 25 + tests/webpack/WORKSPACE | 34 ++ tests/webpack/input.js | 1 + tests/webpack/webpack_bin_test.sh | 8 + tests/webpack/webpack_compile_test.sh | 8 + 37 files changed, 1233 insertions(+), 505 deletions(-) create mode 100755 node/internal/mocha_test.sh create mode 100644 node/internal/node_bundle.bzl create mode 100644 node/internal/node_modules.bzl create mode 100644 tests/helloworld/gen/BUILD create mode 100644 tests/helloworld/gen/compiler.js create mode 100755 tests/helloworld/gen/generated_helloworld_test.sh create mode 100644 tests/helloworld/gen/input.js create mode 100644 tests/helloworld/require_test.js create mode 100644 tests/mocha/src/main/js/BUILD create mode 100644 tests/mocha/src/main/js/greeter.js rename tests/mocha/{tests => src/main/js}/test.js (100%) delete mode 100644 tests/mocha/tests/BUILD create mode 100644 tests/webpack/BUILD create mode 100644 tests/webpack/README.md create mode 100644 tests/webpack/WORKSPACE create mode 100644 tests/webpack/input.js create mode 100755 tests/webpack/webpack_bin_test.sh create mode 100755 tests/webpack/webpack_compile_test.sh diff --git a/.travis.yml b/.travis.yml index 4f4b95e..30ce8e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,10 @@ os: - osx env: + - V=0.7.0 + - V=0.6.1 - V=0.5.4 - V=0.5.3 - # Not compatible under 5.3 before_install: - OS=linux diff --git a/Makefile b/Makefile index 9566191..6dd9581 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,22 @@ -test_all: - (cd tests/helloworld && bazel test //:helloworld_test) - (cd tests/lyrics && bazel test //:lyrics_test) - (cd tests/express && bazel test //:server_test) - (cd tests/namespace && bazel test //:question_test) - (cd tests/typescript && bazel test //:typescript_test) - (cd tests/mocha && bazel test //:test) - (cd tests/mocha && bazel test //tests:test) +test_helloworld: + (cd tests/helloworld && bazel test //...) + +test_lyrics: + (cd tests/lyrics && bazel test //...) + +test_express: + (cd tests/express && bazel test //...) + +test_namespace: + (cd tests/namespace && bazel test //...) + +test_typescript: + (cd tests/typescript && bazel test //...) + +test_webpack: + (cd tests/webpack && bazel test //...) + +test_mocha: + (cd tests/mocha && bazel test //...) + +test_all: test_helloworld test_lyrics test_express test_namespace test_typescript test_webpack test_mocha diff --git a/README.md b/README.md index b896a9a..2613e77 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,11 @@ | Rule | Description | | ---: | :---------- | | [node_repositories](#node_repositories) | Install node toolchain. | -| [yarn_modules](#yarn_modules) | Install a set node_modules dependencies using yarn. | -| [node_module](#node_module) | Define a node module from a set of source files and a main (or index) source file. | -| [node_binary](#node_binary) | Build a node_modules tree and execute an entrypoint module script. | -| [mocha_test](#mocha_test) | Run a mocha test script. | +| [yarn_modules](#yarn_modules) | Install a set node module dependencies using yarn. | +| [node_module](#node_module) | Define a node module from a set of source files (having an optional main (or index) entry point). | +| [node_binary](#node_binary) | Run a node module. | +| [node_test](#node_test) | Run a node binary as a bazel test. | +| [mocha_test](#mocha_test) | Run a mocha test script. | @@ -84,13 +85,14 @@ populate it with the necessary dependencies. 3. Read the generated `yarn.lock` file, parse it, and write out a `@yarn_modules//:BUILD` file. This file contains a `node_module` rule foreach entry in the `yarn.lock` file, a `node_module` rule - with the special name `_all_`, and an `sh_binary` rule foreach + with the special name `_all_`, and a `node_binary` rule foreach executable script in the `node_modules/.bin` folder. > Note 1: You can inspect all the targets by running `bazel query @yarn_modules//:*`. > Note 2: The workspace name `yarn_modules` is arbitrary, choose -whatever you like *other than* `node_modules` (that one doesn't work). +whatever you like (*other than* `node_modules` itself, that one +doesn't work). At this point you can use these rule targets as `deps` for your `node_module` rules. *Example*: @@ -152,7 +154,7 @@ node_modules/fs-extra When used by other `node_module` rules, you can import the module as: ```javascript -const myModule = require("my-module"); +const myModule = require("my_module"); ``` There are three basic ways to create a `node_module` rule: @@ -249,6 +251,7 @@ These are only relevant if you don't explicitly name a `package.json` file. | optional | `string` | `url` | `None` | Url where the module tgz archive was resolved | optional | `string` | `sha1` | `None` | Sha1 hash of of the resolved tgz archive | optional | `string` | `description` | `None` | Module description +| optional | `string_dict` | `executables` | `None` | A mapping from binary name to internal node module path. Example `executables = { 'foo': 'bin/foo' }`. ### node_module attributes that affect the relative path of files included in the module @@ -267,9 +270,7 @@ the workspace, which needs to be preserved in the generated module. ## node_binary -The `node_binary` rule builds a `node_modules/` tree based on its -`node_module` dependencies and writes a script to execute a module -entrypoint. +The `node_binary` rule writes a script to execute a module entrypoint. ```python load("@org_pubref_rules_node//node:rules.bzl", "node_binary") @@ -299,38 +300,55 @@ the entrypoint (under the hood, it will just build a `node_module` becoming equivalent to the first example). +```python +node_binary( + name = "foo", + entrypoint = ":my_module_2", + executable = "baz", +) +``` + +In this third example (above), we're specifying the name of the node +module to start with (`my_module_2`) and the name of the executable +within `my_module_2` to run (`baz`). In this case the `node_module` +rule definition for `my_module_2` must have a `string_dict` with an +entry for `baz` (like `executables = { 'baz': 'bin/baz' }`. + ### Output structure of files generated for a `node_binary` rule -A `node_binary` rule named `foo` will create a folder having exactly two entries: +A `node_binary` rule named `foo` will create a folder having exactly +two entries: 1. An executable shell script named `foo`. -1. A folder which bundles up all the needed files in `foo_bundle/`. +1. A folder which bundles up all the needed files in `foo_files/`. -Within `foo_bundle/`, there will also be exactly two entries: +Within `foo_files/`, there will also be exactly two entries: 1. The `node` executable itself. -1. The `node_modules/` folder with all the built/copied modules. - -The bash shell script `foo` performs the following: - -`cd $(dirname $0)/foo_bundle && exec node node_modules/entrypoint` +1. The `node_modules/` folder with all the built/copied modules + (including the entrypoint module). ### Building a deployable bundle -To generate a tarred gzipped archive of the above example that you can -ship as a single 'executable' file, invoke `$ bazel build -:{target}_bundle.tgz`. This is similar in intent to the java +To generate a tarred/gzipped archive of the above example that you can +ship as a single 'executable' self-contained package, invoke `$ bazel +build :{target}_deploy.tar.gz`. This is similar in intent to the java `{target}_deploy.jar` implicit build rule. ```sh -$ bazel build :foo_bundle.tgz -Target //:foo_bundle.tgz up-to-date: +$ bazel build :foo_deploy +Target //:foo_deploy.tar.gz up-to-date: bazel-bin/foo_bundle.tgz $ du -h bazel-bin/foo_bundle.tgz 33M bazel-bin/foo_bundle.tgz ``` +## node_test + +The `node_test` rule is identical to node_binary, but sets the `test = +True` flag such that it can be used as a bazel test. + ## mocha_test Runs a mocha test identified by the start script given in `main` or @@ -353,13 +371,9 @@ mocha_test( name = "test", main = "test.js", ) - -mocha_test( - name = "test", - entrypoint = ":my_module", -) ``` ## Conclusion -That's it! Please refer to the various workspaces in `tests/` and the source for more detail. +That's it! Please refer to the various workspaces in `tests/` and the +source for more detail. diff --git a/WORKSPACE b/WORKSPACE index a1ac749..cd62356 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,3 +3,4 @@ workspace(name = "org_pubref_rules_node") load("//node:rules.bzl", "node_repositories") node_repositories() + diff --git a/node/BUILD b/node/BUILD index ffd0fb0..aa14f77 100644 --- a/node/BUILD +++ b/node/BUILD @@ -1 +1,5 @@ package(default_visibility = ["//visibility:public"]) + +exports_files([ + "internal/mocha_test.sh", +]) diff --git a/node/internal/mocha_test.bzl b/node/internal/mocha_test.bzl index e46f389..680286f 100644 --- a/node/internal/mocha_test.bzl +++ b/node/internal/mocha_test.bzl @@ -1,104 +1,56 @@ load("//node:internal/node_module.bzl", "node_module") -load("//node:internal/node_binary.bzl", "copy_modules", "binary_attrs") - - -def _create_launcher(ctx, output_dir, node, mocha): - entry_module = ctx.attr.entrypoint.node_module - # if package is under root - entrypoint = '%s_test/node_modules/%s' % (ctx.label.name, entry_module.name) - # if test is under inner package - if ctx.label.package: - entrypoint = '%s/%s' % (ctx.label.package, entrypoint) - cmd = [ - node.short_path, - ] + ctx.attr.node_args + [ - mocha.short_path, - ] + ctx.attr.mocha_args + [ - entrypoint, - ] + ctx.attr.script_args + [ - '$@', - ] - - lines = [ - '#!/usr/bin/env bash', - 'set -e', - ' '.join(cmd) - ] - ctx.file_action( - output = ctx.outputs.executable, - executable = True, - content = '\n'.join(lines), +load("//node:internal/node_modules.bzl", "node_modules") + +def mocha_test( + name = None, + # Main test script entrypoint + main = None, + # Additional module deps for the test + deps = [], + # The mocha binary executable target + mocha_bin = "@mocha_modules//:mocha_bin", + # The script runner for the sh_test + script = "@org_pubref_rules_node//node:internal/mocha_test.sh", + # Any additional args to pass directly to mocha + args = [], + # Test size + size = "small", + # Test visibility + visibility = None, + # Remainder of args go to 'node_module' + **kwargs): + + """Given a rule name and a main test script entrypoint file, package + that test script up as a module, then package that module in a + node_modules tree. Run a bash script that invokes the mocha_bin + executable with the name of the testable entrypoint module. + + """ + + node_module( + name = name + "_module", + main = main, + visibility = visibility, + **kwargs ) - -def mocha_test_impl(ctx): - output_dir = ctx.label.name + '_test' - node = ctx.executable._node - mocha = ctx.executable._mocha_bin - - all_deps = ctx.attr.deps + [ctx.attr.entrypoint] - files = copy_modules(ctx, output_dir, all_deps) - - _create_launcher(ctx, output_dir, node, mocha) - - mocha_deps_all = ctx.attr._mocha_deps.node_module - transitive_mocha_files = mocha_deps_all.files.to_list() - for dep in mocha_deps_all.transitive_deps: - transitive_mocha_files += dep.files.to_list() - - runfiles = [ - node, - mocha, - ctx.outputs.executable - ] + transitive_mocha_files + files - - return struct( - runfiles = ctx.runfiles( - files = runfiles, - collect_data = True, - ), + node_modules( + name = name + "_modules", + target = name + "_modules", + visibility = visibility, + deps = deps + [name + "_module"], ) - - -_mocha_test = rule( - mocha_test_impl, - attrs = binary_attrs + { - "_mocha_bin": attr.label( - default = Label("@mocha_modules//:mocha_bin"), - allow_files = True, - executable = True, - cfg = "host", - ), - "_mocha_deps": attr.label( - providers = ["node_module"], - default = Label("@mocha_modules//:_all_"), - ), - "mocha_args": attr.string_list(), - }, - test = True, -) - - -def mocha_test(name = None, main = None, entrypoint = None, node_args = [], mocha_args = [], deps = [], visibility = None, size = "small", **kwargs): - - if not entrypoint: - if not main: - fail('Either an entrypoint node_module or a main script file must be specified') - entrypoint = name + '_module' - node_module( - name = entrypoint, - main = main, - deps = [], - visibility = visibility, - **kwargs - ) - - _mocha_test( + + native.sh_test( name = name, - entrypoint = entrypoint, - deps = deps, + srcs = [script], + args = args + [ + "{name}_modules/node_modules/{name}_module".format(name = name), + ], + data = [ + mocha_bin, + name + "_modules", + ], size = size, - node_args = node_args, - mocha_args = mocha_args, visibility = visibility, ) diff --git a/node/internal/mocha_test.sh b/node/internal/mocha_test.sh new file mode 100755 index 0000000..a1217df --- /dev/null +++ b/node/internal/mocha_test.sh @@ -0,0 +1,2 @@ +set -eu +external/mocha_modules/mocha_bin $@ diff --git a/node/internal/node_binary.bzl b/node/internal/node_binary.bzl index 542940f..391bacf 100644 --- a/node/internal/node_binary.bzl +++ b/node/internal/node_binary.bzl @@ -1,94 +1,88 @@ -load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("//node:internal/node_modules.bzl", "node_modules") load("//node:internal/node_module.bzl", "node_module") +load("//node:internal/node_bundle.bzl", "node_bundle") -_node_filetype = FileType(['.js', '.node']) -def _get_relative_dirname(file): - return file.path[0:-len(file.short_path)] +def create_launcher(ctx, output_dir, node, manifest): + """Create a launch bash script + Given the ctx object, the name of the target output dir, the node + executable, and a manifest file (whose 'dirname' points to the + node_modules location), write out a script that runs node. It + looks for several directory layout patterns that may exist if + invoked under various conditions (bazel run, bazel test, genrule, + standalone script, within a standalone bundle). -def _get_filename_relative_to_module(module, file): - name = module.name - parts = file.path.partition(name) - return '/'.join(parts[1:]) + """ - -def _copy_module(ctx, output_dir, module): - if len(module.files) == 0: - return [] - - inputs = [] - outputs = [] - - script_file = ctx.new_file('%s/copy_%s.sh' % (output_dir, module.identifier)) - script_lines = [] - - for src in module.files: - inputs.append(src) - dst_filename = _get_filename_relative_to_module(module, src) - dst = ctx.new_file('%s/node_modules/%s' % (output_dir, dst_filename)) - outputs.append(dst) - script_lines.append("cp '%s' '%s'" % (src.path, dst.path)) - - ctx.file_action( - output = script_file, - content = '\n'.join(script_lines), - executable = True, - ) - - ctx.action( - mnemonic = 'CopyModuleWith%sFiles' % len(outputs), - inputs = inputs + [script_file], - outputs = outputs, - command = script_file.path, - ) - - return outputs - -# NOTE(pcj): I tried in vain to make a version of this based on -# symlinks, either of the folders or the files themselves. Maybe you -# can get that figured out. -def copy_modules(ctx, output_dir, deps): - outputs = [] - for dep in deps: - module = dep.node_module - outputs += _copy_module(ctx, output_dir, module) - for module in module.transitive_deps: - outputs += _copy_module(ctx, output_dir, module) - return outputs - - -def _create_launcher(ctx, output_dir, node): entry_module = ctx.attr.entrypoint.node_module - entrypoint = 'node_modules/%s' % entry_module.name - - # cd $(dirname $0)/bundle and exec node node_modules/foo + + # Entrypoint is a string that is used in the script for the node + # main start point. It can be the name of the module itself (in + # which case node looks for index.js or package.json), or an + # executable 'bin' script path within the module. + entrypoint = entry_module.name + if ctx.attr.executable: + entrypoint += "/" + entry_module.executables[ctx.attr.executable] + + # The package path is the path + package_path = [] + if ctx.label.workspace_root: + package_path.append(ctx.label.workspace_root) + if ctx.label.package: + package_path.append(ctx.label.package) + cmd = [ - 'cd $ROOT/%s' % output_dir, - '&&', - 'exec', - ctx.executable._node.basename, + # Replace shell process with node process + 'exec "${TARGET_PATH}${NODE_EXE}"', ] + ctx.attr.node_args + [ - entrypoint, + '"${TARGET_PATH}node_modules/${ENTRYPOINT}"', ] + ctx.attr.script_args + [ '$@', ] lines = [ - '#!/usr/bin/env bash', # TODO(user): fix for windows - 'set -e', - - # Set the execution root to the same directory where the - # script lives. We know for sure that node executable and - # node_modules dir will also be close to here since we - # specifically built that here (this means we don't have to go - # through backflips to figure out what run context we're in. - 'ROOT=$(dirname $0)', - - # Resolve to this node instance if other scripts have - # '/usr/bin/env node' shebangs - # TODO: fix for windows - 'export PATH="$ROOT:$PATH"', + '#!/usr/bin/env bash', + 'set -eu', # -eux for debugging + + # Location of node + 'NODE_EXE="%s"' % node.basename, + # Path to the node module + 'PACKAGE_PATH="%s"' % "/".join(package_path), + # Namespace where node and node_modules assets have been + # built. + 'TARGET_NAME="%s"' % ctx.attr.target, + # Based on the way the script has been invoked, $PACKAGE_PATH + # can exist in various locations. We need to discover this + # and assign the value to $TARGET_PATH. + 'TARGET_PATH=""' , + + 'ENTRYPOINT="%s"' % entrypoint, + + 'if [[ -e "${0}_files/${NODE_EXE}" ]]; then', + ' TARGET_PATH="${0}_files/"', + #' echo "Matched [standalone script] or [bazel test] context [${TARGET_PATH}]"', + '', + 'elif [[ -e "${0}_files/${NODE_EXE}" ]]; then', + ' TARGET_PATH="${0}_files/"', + #' echo "Matched [standalone script] or [bazel test] context [${TARGET_PATH}]"', + '', + 'elif [[ -e "${0}.runfiles/__main__/${PACKAGE_PATH}/${TARGET_NAME}/${NODE_EXE}" ]]; then', + ' TARGET_PATH="${0}.runfiles/__main__/${PACKAGE_PATH}/${TARGET_NAME}/"', + #' echo "Matched [bazel run] context [${TARGET_PATH}]"', + '', + 'elif [[ -e "${PACKAGE_PATH}/${TARGET_NAME}/${NODE_EXE}" ]]; then', + ' TARGET_PATH="${PACKAGE_PATH}/${TARGET_NAME}/"', + #' echo "Matched [bazel test TESTRULE] context [${TARGET_PATH}]"', + '', + 'else', + ' echo "Failed to find target execution path! Aborting" >&2', + ' exit 1', + 'fi', + + '# Modify path such that embedded scripts with /usr/bin/env node shebangs', + '# resolve to the bundled node executable', + 'export PATH="${TARGET_PATH}:$PATH"', ' '.join(cmd) ] @@ -99,41 +93,23 @@ def _create_launcher(ctx, output_dir, node): content = '\n'.join(lines), ) - + def node_binary_impl(ctx): - output_dir = ctx.label.name + '_bundle' - - manifest_file = ctx.new_file('%s/node_modules/manifest.json' % output_dir) - json = {} - all_deps = ctx.attr.deps + [ctx.attr.entrypoint] - files = copy_modules(ctx, output_dir, all_deps) - - dependencies = {} - for dep in all_deps: - module = dep.node_module - dependencies[module.name] = module.version - json['dependencies'] = struct(**dependencies) - - manifest_content = struct(**json) - - ctx.file_action( - output = manifest_file, - content = manifest_content.to_json(), - ) + target_dir = ctx.attr.target - node = ctx.new_file('%s/%s' % (output_dir, ctx.executable._node.basename)) + node = ctx.new_file('%s/%s' % (target_dir, ctx.executable._node.basename)) ctx.action( mnemonic = 'CopyNode', inputs = [ctx.executable._node], outputs = [node], command = 'cp %s %s' % (ctx.executable._node.path, node.path), ) + + files = ctx.attr.node_modules.node_modules.files + create_launcher(ctx, target_dir, node, ctx.attr.node_modules.node_modules.manifest) - _create_launcher(ctx, output_dir, node) - - runfiles = [node, manifest_file, ctx.outputs.executable] + files - files = runfiles if ctx.attr.export_files else [] - + runfiles = [node, ctx.outputs.executable] + files + return struct( runfiles = ctx.runfiles( files = runfiles, @@ -141,21 +117,40 @@ def node_binary_impl(ctx): ), node_binary = struct( files = runfiles, + node = node, ) ) binary_attrs = { + # The main entrypoint module to run 'entrypoint': attr.label( providers = ['node_module'], + mandatory = False, + ), + # An optional named executable module script to run + 'executable': attr.string( + mandatory = False, + ), + # node_module dependencies. Given the entrypoint module must + # exist within the node_modules tree, this is a mandatory + # attribute. + 'node_modules': attr.label( mandatory = True, + providers = ['node_modules'], ), - 'deps': attr.label_list( - providers = ['node_module'], + # A namespace (string) within the package where assets are built. + # This attribute value is also passed the node_modules such that + # everything gets built in the same location. + 'target': attr.string( + mandatory = True, ), + # Raw Arguments to the node executable 'node_args': attr.string_list( ), + # Arguments to be included in the launcher script 'script_args': attr.string_list( ), + # The node executable '_node': attr.label( default = Label('@node//:node'), single_file = True, @@ -168,31 +163,20 @@ binary_attrs = { _node_binary = rule( node_binary_impl, - attrs = binary_attrs + { - 'export_files': attr.bool( - default = False, - ), - }, + attrs = binary_attrs, executable = True, ) -def node_binary_files_impl(ctx): - return struct( - files = depset(ctx.attr.target.node_binary.files), - ) - -_node_binary_files = rule( - node_binary_files_impl, - attrs = { - 'target': attr.label( - providers = ['node_binary'], - mandatory = True, - ), - }, -) - -def node_binary(name = None, main = None, entrypoint = None, version = None, node_args = [], deps = [], extension = 'tgz', visibility = None, **kwargs): +def node_binary(name = None, + main = None, + entrypoint = None, + executable = None, + node_args = [], + deps = [], + deploy = 'tar.gz', + visibility = None, + **kwargs): if not entrypoint: if not main: @@ -201,32 +185,78 @@ def node_binary(name = None, main = None, entrypoint = None, version = None, nod node_module( name = entrypoint, main = main, - deps = [], - version = version, visibility = visibility, **kwargs ) + node_modules( + name = name + '_modules', + target = name + '_files', + deps = deps + [entrypoint], + visibility = visibility, + ) + _node_binary( name = name, + target = name + '_files', entrypoint = entrypoint, - deps = deps, - export_files = name.endswith('_bundle.tgz'), + executable = executable, + node_modules = name + '_modules', node_args = node_args, visibility = visibility, ) - _node_binary_files( - name = name + '_files', - target = name, + node_bundle( + name = name + '_bundle', + node_binary = name, + extension = deploy, visibility = visibility, ) - pkg_tar( - name = name + '_bundle', - extension = extension, - package_dir = name, - srcs = [name + '_files'], + +_node_test = rule( + node_binary_impl, + attrs = binary_attrs, + test = True, +) + + +def node_test(name = None, + main = None, + entrypoint = None, + executable = None, + node_args = [], + deps = [], + size = None, + visibility = None, + **kwargs): + + if not entrypoint: + if not main: + fail('Either an entrypoint node_module or a main script file must be specified') + entrypoint = name + '_module' + node_module( + name = entrypoint, + main = main, + deps = deps, + visibility = visibility, + **kwargs + ) + + + node_modules( + name = name + '_modules', + deps = deps + [entrypoint], + target = name + '_files', + ) + + _node_test( + name = name, + entrypoint = entrypoint, + executable = executable, + node_modules = name + '_modules', + target = name + '_files', + node_args = node_args, visibility = visibility, - strip_prefix = '.', + size = size, ) diff --git a/node/internal/node_bundle.bzl b/node/internal/node_bundle.bzl new file mode 100644 index 0000000..4a7610b --- /dev/null +++ b/node/internal/node_bundle.bzl @@ -0,0 +1,37 @@ +load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") + +def _node_bundle_impl(ctx): + return struct( + files = depset(ctx.attr.node_binary.node_binary.files), + ) + +_node_bundle = rule( + _node_bundle_impl, + attrs = { + 'node_binary': attr.label( + providers = ['node_binary'], + mandatory = True, + ), + }, +) + + +def node_bundle(name = None, + node_binary = None, + extension = None, + visibility = None, + **kwargs): + + _node_bundle( + name = name, + node_binary = node_binary + ) + + pkg_tar( + name = node_binary + '_deploy', + extension = extension, + package_dir = name, + srcs = [name], + visibility = visibility, + strip_prefix = './', + ) diff --git a/node/internal/node_module.bzl b/node/internal/node_module.bzl index f98a51b..b73aa62 100644 --- a/node/internal/node_module.bzl +++ b/node/internal/node_module.bzl @@ -1,25 +1,34 @@ _node_filetype = FileType([".js", ".node"]) + def _relname(ctx, root_file, file): - #print("getting relative name for %s rel %s" % (file.path, root_file.path)) - # If file is in the workspace root, just return the name - if file.dirname == ".": - return file.short_path - parts = file.path.partition(root_file.dirname) - # If the file.path does not contain root_file.dirname, try the - # label.package... - if not len(parts[2]): - # However, if the label.package is empty, file is in the - # workspace root (so just use the basename) - if not ctx.label.package: - return file.basename - parts = file.path.partition(ctx.label.package) - if not len(parts[2]): - print("failed relative name for %s rel %s" % (file.path, root_file.path)) - return parts[2] + """Get a path string relative to the root_file (usually the + package.json file of the module). + + """ + + # For files that are in the external/ namespace, just slice off + # the root_file dirname. + if file.path.startswith("external/") and file.path.startswith(root_file.dirname): + rel = file.path[len(root_file.dirname):] + return rel + + + if not file.short_path.startswith(ctx.label.package): + fail("_relname failed: %s must start with %s" % (file.short_path, ctx.label.package)) + + # All others, shave off the package label from the file's short + # path. + rel = file.short_path[len(ctx.label.package):] + if rel.startswith('/'): + rel = rel[1:] + + return rel def _get_package_dependencies(module_deps): + """ Create a mapping like { "underscore": "1.8.3", ... } + """ dependencies = {} for dep in module_deps: module = dep.node_module @@ -35,12 +44,12 @@ def _get_module_name(ctx): # else use the package name, but only if non-empty elif ctx.label.package: parts += ctx.label.package.split("/") - # finally, use the module_name or label name + # finally, add the module_name or label name parts.append(ctx.attr.module_name or ctx.label.name) return ctx.attr.separator.join(parts) -def _create_package_json(ctx, name, files): +def _create_package_json(ctx, name, files, executables): output_file = ctx.new_file("%s/package.json" % name) json = { @@ -52,8 +61,11 @@ def _create_package_json(ctx, name, files): } if len(files) > 0: - json["files"] = list(depset([_get_path_for_module_file(ctx, output_file, file, {}) for file in files])) + json["files"] = depset([_get_path_for_module_file(ctx, output_file, file, {}) for file in files]).to_list() + if executables: + json["bin"] = executables + if ctx.attr.main: json["main"] = ctx.file.main.basename @@ -78,7 +90,7 @@ def _get_transitive_modules(deps, key): for dep in deps: module = dep.node_module modules += [module] - modules += getattr(module, key) + modules += getattr(module, key, []) return modules @@ -119,6 +131,8 @@ def _node_module_impl(ctx): if ctx.file.main: files.append(ctx.file.main) + executables = ctx.attr.executables + package_json = ctx.file.package_json # The presence of an index file suppresses creation of the @@ -126,7 +140,7 @@ def _node_module_impl(ctx): # provided. if len(files) > 0 and not package_json: if ctx.attr.main or not ctx.file.index: - package_json = _create_package_json(ctx, name, files) + package_json = _create_package_json(ctx, name, files, executables) if package_json: outputs.append(package_json) @@ -154,6 +168,7 @@ def _node_module_impl(ctx): url = ctx.attr.url, sha1 = ctx.attr.sha1, description = ctx.attr.description, + executables = executables, package_json = package_json, root = root_file, sourcemap = sourcemap, @@ -198,7 +213,7 @@ node_module = rule( # File.short_path, causing them to be relative to the # WORKSPACE. "layout": attr.string( - values = ["relative", "workspace"], + values = ["relative", "workspace", "flat"], default = "relative", ), @@ -232,6 +247,13 @@ node_module = rule( providers = ["node_module"], ), + # 'Binary' scripts, to be named in the 'package_json.bin' + # property. This uses the plain 'string_dict' attribute since + # bazel does not have the more intuitive + # 'string_keyed_label_dict'-type attribute. + "executables": attr.string_dict( + ), + # Module version "version": attr.string( default = "1.0.0", @@ -254,7 +276,6 @@ node_module = rule( # File that should be named as the package.json 'main' # attribute. "main": attr.label( - #allow_files = _node_filetype, allow_files = True, mandatory = False, single_file = True, diff --git a/node/internal/node_modules.bzl b/node/internal/node_modules.bzl new file mode 100644 index 0000000..f09f526 --- /dev/null +++ b/node/internal/node_modules.bzl @@ -0,0 +1,102 @@ + +def _get_filename_relative_to_module(module, file): + parts = file.path.partition("/%s/" % module.name) + return '/'.join(parts[1:]) + + +def _copy_module(ctx, target_dir, module): + if len(module.files) == 0: + return [] + + inputs = [] + outputs = [] + + script_file = ctx.new_file('%s/copy_%s.sh' % (ctx.label.name, module.identifier)) + #script_file = ctx.new_file('copy_%s.sh' % (module.identifier)) + script_lines = [] + + for src in module.files: + inputs.append(src) + dst_filename = _get_filename_relative_to_module(module, src) + dst = ctx.new_file('%s/node_modules/%s' % (target_dir, dst_filename)) + outputs.append(dst) + script_lines.append("cp '%s' '%s'" % (src.path, dst.path)) + + ctx.file_action( + output = script_file, + content = '\n'.join(script_lines), + executable = True, + ) + + ctx.action( + mnemonic = 'CopyModuleWith%sFiles' % len(outputs), + inputs = inputs + [script_file], + outputs = outputs, + command = script_file.path, + ) + + return outputs + + +def _copy_modules(ctx, target_dir, deps): + outputs = [] + for dep in deps: + module = dep.node_module + outputs += _copy_module(ctx, target_dir, module) + for module in module.transitive_deps: + outputs += _copy_module(ctx, target_dir, module) + return outputs + + +def node_modules_impl(ctx): + #target_dir = ctx.label.name + '.node_modules' + target_dir = ctx.attr.target + files = _copy_modules(ctx, target_dir, ctx.attr.deps) + manifest = ctx.new_file('%s/node_modules/manifest.json' % target_dir) + files.append(manifest) + + modules = [] + for dep in ctx.attr.deps: + module = dep.node_module + modules.append(module) + for m in module.transitive_deps: + modules.append(m) + + json = {} + dependencies = {} + for module in modules: + dependencies[module.name] = module.version + json['dependencies'] = struct(**dependencies) + + manifest_content = struct(**json) + + ctx.file_action( + output = manifest, + content = manifest_content.to_json(), + ) + + return struct( + files = depset(files), + runfiles = ctx.runfiles( + files = files, + collect_data = True, + ), + node_modules = struct( + files = files, + modules = depset(modules), + manifest = manifest, + ) + ) + + +node_modules = rule( + node_modules_impl, + attrs = { + 'deps': attr.label_list( + providers = ['node_module'], + ), + 'target': attr.string( + mandatory = True, + ), + } +) diff --git a/node/internal/node_repositories.bzl b/node/internal/node_repositories.bzl index 35be37a..03c5a3b 100644 --- a/node/internal/node_repositories.bzl +++ b/node/internal/node_repositories.bzl @@ -1,4 +1,4 @@ -# The node_repository_impl taken from Alex Eagle's rules_nodejs :) +# The node_repository_impl is mostly taken from rules_nodejs :) # # Copyright 2017 The Bazel Authors. All rights reserved. # diff --git a/node/internal/parse_yarn_lock.js b/node/internal/parse_yarn_lock.js index 884dad7..c341cf8 100644 --- a/node/internal/parse_yarn_lock.js +++ b/node/internal/parse_yarn_lock.js @@ -2,36 +2,233 @@ const fs = require('fs'); const lockfile = require('@yarnpkg/lockfile'); +const path = require('path') -let file = fs.readFileSync('yarn.lock', 'utf8'); -let json = lockfile.parse(file); +main(); -if (json.type !== 'success') { - throw new Error('Lockfile parse failed: ' + JSON.stringify(json, null, 2)); +/** + * Main entrypoint. Write to stdout that will be captured into a + * BUILD file. + */ +function main() { + // Read the yarn.lock file and parse it. + // + let file = fs.readFileSync('yarn.lock', 'utf8'); + let yarn = lockfile.parse(file); + + if (yarn.type !== 'success') { + throw new Error('Lockfile parse failed: ' + JSON.stringify(yarn, null, 2)); + } + + // Foreach entry in the lockfile, create an entry object. We'll + // supplement/merge this with information from the package.json file + // in a moment... + // + const entries = Object.keys(yarn.object).map(key => makeYarnEntry(key, yarn.object[key])); + + // For all top-level folders in the node_modules directory that + // contain a package.json file... + const getModulesIn = p => fs.readdirSync(p) + .filter(f => + fs.statSync(path.join(p, f)).isDirectory() && + fs.existsSync(path.join(p, f, 'package.json')) && + fs.statSync(path.join(p, f, 'package.json')).isFile()); + + // ... parse ithem + const modules = getModulesIn('node_modules').map(dir => parseNodeModulePackageJson(dir)); + + // Iterate all the modules and merge the information from yarn into + // the module + modules.forEach(module => mergePackageJsonWithYarnEntry(entries, module)); + + // Didn't realize that the nodejs module ecosystem can contain + // circular references, but apparently it can. + breakCircularDependencies(modules) + + // Print output + // + print(""); + print("package(default_visibility = ['//visibility:public'])"); + print("load('@org_pubref_rules_node//node:rules.bzl', 'node_module', 'node_binary')"); + + modules.forEach(module => printNodeModule(module)); + + printNodeModuleAll(modules); + + // Create an executable rule all executable entryies in the modules + modules.forEach(module => { + if (module.executables) { + for (const [name, path] of module.executables.entries()) { + printNodeBinary(module, name, path); + } + } + }); + + print(""); + print("# EOF"); +} + + +/** + * Given a list of yarn entries and a target module, find an exact + * match by name and version. + */ +function findMatchingYarnEntryByNameAndVersion(entries, module) { + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + if (entry.name === module.name && entry.version === module.version) { + return entry; + } + } +} + + +/** + * Given a list of yarn entries and a target module, merge them. + * Actually, this is pretty simple as the yarn entry is simply + * attached to the module. + */ +function mergePackageJsonWithYarnEntry(entries, module) { + const entry = findMatchingYarnEntryByNameAndVersion(entries, module); + if (!entry) { + throw new Error("No matching node_module found for " + module.name); + } + + // Use the bazelified name as the module name + module.original_name = module.name + module.name = entry.name + // Store everything else here + module.yarn = entry; } -const entries = Object.keys(json.object).map(key => makeEntry(key, json.object[key])); -const cache = new Map(); +/** + * Given a list of modules, build a graph of their dependencies and + * collapse it according to Tarjan's SCC algorithm. For any strongly + * connected components, break out the cluster into it's own module + * and rewrite the dependency graph to point to the cluster rather + * than the individual module entries. + */ +function breakCircularDependencies(modules) { + + const byName = new Map(); + modules.forEach(module => byName.set(module.name, module)); + + // Make a list of nodes + const nodes = Array.from(byName.keys()); + // An Array> array for the edges + const edges = []; + // And a mapping for backreferences mapped by name + const backrefs = new Map(); + + // Build the adjacencyList + nodes.forEach((node, index) => { + const list = []; + edges[index] = list; + const entry = byName.get(node); + // Make a set of deps rather than using the entry.dependencies + // mapping. + entry.deps = new Set(); + + if (entry.dependencies) { + + Object.keys(entry.dependencies).forEach(name => { -print(""); -print("package(default_visibility = ['//visibility:public'])"); -print("load('@org_pubref_rules_node//node:rules.bzl', 'node_module', 'node_binary')"); + // Save this in the deps set + const dependency = byName.get(name); + entry.deps.add(dependency); + + // Populate the adjacency list + const depIndex = nodes.indexOf(name); + list.push(depIndex); + + // Compute referrer backreferences for later use. + let referrer = dependency.referrer; + if (!referrer) { + referrer = dependency.referrer = new Set(); + } + referrer.add(entry); -entries.forEach(entry => printNodeModule(entry)); + }); + } + }); -printNodeModules(cache); + const clusters = stronglyConnectedComponents(edges); + + // Foreach non-trivial cluster in the SCC, create a pseudo-module + // for the cluster and re-link each entry to point to the cluster + // rather than the dependency. + clusters.components.forEach((component, index) => { + + if (component.length > 1) { + // console.log("SCC: ", component); + // component.forEach(element => { + // console.log(`Component ${index} contains ${nodes[element]} (${element})`); + // }); + + // Create a name for the pseudo-module + const name = '_scc' + index; + // The dependencies in this cluster component + const deps = new Set(); + // The pseudo-module for the cluster + const pseudo = { + name: name, + deps: deps + }; + + // A list of entries in this component + const list = []; + // Last entry in the component can be standalone + for (let i = 0; i < component.length; i++) { + list.push(byName.get(nodes[component[i]]) ); + } -cache.forEach(entry => parsePackageJson(entry)); + // A description for the module + pseudo.description = "Strongly connected component containing " + list.map(e => e.name).join(", ") + + list.forEach(entry => { + // Add this to the pseudo-module + deps.add(entry); -print(""); -print("# EOF"); + // Iterate the set of items that link to this entry. Replace + // their deps set with the psudo-module rather than the entry + // itself. + entry.referrer.forEach(ref => { + ref.deps.delete(entry); -function makeEntry(key, entry) { + // Add an entry to the scc component (unless it is a member + // of it). + if (!deps.has(ref)) { + ref.deps.add(pseudo); + } + }); + + // Each entry in the cluster must have no other outgoing + // dependencies + entry.deps = new Set(); + }); + + // Store this new pseudo-module in the modules list + modules.push(pseudo); + } + + }); + +} + +/** + * Given an entry from lockfile.parse, do additional processing to + * assign the name and version. + */ +function makeYarnEntry(key, entry) { parseName(key, entry); parseResolved(entry); return entry; } + +/** + * Parse a yarn name into something that will be agreeable to bazel. + */ function parseName(key, entry) { // can be 'foo@1.0.0' or something like '@types/foo@1.0.0' const at = key.lastIndexOf('@'); @@ -42,6 +239,10 @@ function parseName(key, entry) { entry.label = label; } + +/** + * Parse the yarn 'resolved' entry into its component url and sha1. + */ function parseResolved(entry) { const resolved = entry.resolved; if (resolved) { @@ -51,109 +252,260 @@ function parseResolved(entry) { } } -function printDownloadMeta(entry) { - print("# <-- " + [entry.sha1,entry.name,entry.url].join("|")); -} - + +/** + * Reformat/pretty-print a json object as a skylark comment (each line + * starts with '# '). + */ function printJson(entry) { + // Hacky workaround to avoic circular issues when JSONifying + const deps = entry.deps; + const referrer = entry.referrer; + JSON.stringify(entry, null, 2).split("\n").forEach(line => print("# " + line)); + + entry.deps = deps; + entry.referrer = referrer; } -function printNodeModule(entry) { + +/** + * Given a module, print a skylark `node_module` rule. + */ +function printNodeModule(module) { + const deps = module.deps; + print(``); - printJson(entry); - const prev = cache.get(entry.name); - if (prev) { - print(`## Skipped ${entry.id} (${entry.name} resolves to ${prev.id})`); - return; - } + printJson(module); print(`node_module(`); - print(` name = "${entry.name}",`); - print(` version = "${entry.version}",`); - print(` url = "${entry.url}",`); - print(` sha1 = "${entry.sha1}",`); - print(` package_json = "node_modules/${entry.name}/package.json",`); - print(` srcs = glob(["node_modules/${entry.name}/**/*"], exclude = ["node_modules/${entry.name}/package.json"]),`); - - if (entry.dependencies) { + print(` name = "${module.name}",`); + + // SCC pseudomodule wont have 'yarn' property + if (module.yarn) { + const url = module.yarn.url || module.url; + const sha1 = module.yarn.sha1; + const executables = module.executables; + + print(` version = "${module.version}",`); + print(` package_json = "node_modules/${module.name}/package.json",`); + print(` srcs = glob(["node_modules/${module.name}/**/*"], exclude = ["node_modules/${module.name}/package.json"]),`); + if (url) { + print(` url = "${url}",`); + } + if (sha1) { + print(` sha1 = "${sha1}",`); + } + + if (executables.size > 0) { + print(` executables = {`); + for (const [name, val] of executables.entries()) { + print(` "${name}": "${val}",`); + } + print(` },`); + } + + } + if (deps && deps.size) { print(` deps = [`); - Object.keys(entry.dependencies).forEach(module => { - print(` ":${module}",`); + deps.forEach(dep => { + print(` ":${dep.name}",`); }); print(` ],`); } print(`)`); - - cache.set(entry.name, entry); } -function printNodeModules(map) { + +/** + * Given a list of modules, print a skylark `node_module` rule that + * exports all its deps. + */ +function printNodeModuleAll(modules) { print(``); print(`# Pseudo-module that basically acts as a module collection for the entire set`); print(`node_module(`); print(` name = "_all_",`); print(` deps = [`); - for (let entry of map.values()) { - print(` ":${entry.name}",`); - } + modules.forEach(module => { + print(` ":${module.name}",`); + }); print(` ],`); print(`)`); } -function parsePackageJson(entry) { - const pkg = require(`./node_modules/${entry.name}/package.json`); - if (Array.isArray(pkg.bin)) { - // should not happen: throw new Error('Hmm, I didn\'t realize pkg.bin could be an array.'); - } else if (typeof pkg.bin === 'string') { - printNodeModuleShBinary(entry, pkg, entry.name, pkg.bin); - } else if (typeof pkg.bin === 'object') { - Object.keys(pkg.bin).forEach(key => printNodeModuleShBinary(entry, pkg, key, pkg.bin[key])); - } -} -function printNodeModuleShBinary(entry, pkg, name, path) { +/** + * Given a module and the name of an executable defined in it's 'bin' + * property, print a skylark `node_binary` rule. + */ +function printNodeBinary(module, key, path) { + const name = module.name === key ? key : `${module.name}_${key}`; print(``); - print(`sh_binary(`); - print(` name = "${name}_bin",`); // dont want sh_binary 'mkdirp' to conflict - print(` srcs = [":node_modules/.bin/${name}"],`); - print(` data = [`); - print(` ":${entry.name}",`); // must always depend on self - if (pkg.dependencies) { - Object.keys(pkg.dependencies).forEach(dep_name => { - const dep_entry = cache.get(dep_name); - if (!dep_entry) { - throw new Error('Cannot find dependency entry for ' + dep_name); - } - print(` ":${dep_entry.name}",`); - }); - } - print(` ],`); + print(`node_binary(`); + print(` name = "${name}_bin",`); + print(` entrypoint = ":${module.name}",`); + print(` executable = "${key}", # Refers to './${path}' inside the module`); print(`)`); } -function printNodeModuleBinary(entry, pkg, name, path) { - if (path.indexOf("./") === 0) { - path = path.slice(2); + +/** + * Given the name of a top-level folder in node_modules, parse the + * package json and return it as an object. + */ +function parseNodeModulePackageJson(name) { + const module = require(`../node_modules/${name}/package`); + + // Take this opportunity to cleanup the module.bin entries + // into a new Map called 'executables' + const executables = module.executables = new Map(); + + if (Array.isArray(module.bin)) { + // should not happen, but ignore it if present + } else if (typeof module.bin === 'string') { + executables.set(name, stripBinPrefix(module.bin)); + } else if (typeof module.bin === 'object') { + for (let key in module.bin) { + executables.set(key, stripBinPrefix(module.bin[key])); + } } - print(``); - print(`sh_binary(`); - print(` name = "${entry.name}_${name}",`); - print(` srcs = [":node_modules/${entry.name}/${path}"],`); - print(` data = [`); - print(` ":${entry.name}",`); // must always depend on self - if (pkg.dependencies) { - Object.keys(pkg.dependencies).forEach(dep_name => { - const dep_entry = cache.get(dep_name); - if (!dep_entry) { - throw new Error('Cannot find dependency entry for ' + dep_name); - } - print(` ":${dep_entry.name}",`); - }); + + return module; +} + +/** + * Given a path, remove './' if it exists. + */ +function stripBinPrefix(path) { + // Bin paths usually come in 2 flavors: './bin/foo' or 'bin/foo', + // sometimes other stuff like 'lib/foo'. Remove prefix './' if it + // exists. + if (path.indexOf('./') === 0) { + path = path.slice(2); } - print(` ],`); - print(`)`); + return path; } +/** + * Write a string to stdout (console.log). + */ function print(msg) { console.log(msg); } + +/** + * Given an adjacency list, compute Tarjan's SCC. + * + * https://github.com/mikolalysenko/strongly-connected-components/blob/master/scc.js + * Copyright https://github.com/mikolalysenko + */ +function stronglyConnectedComponents(adjList) { + var numVertices = adjList.length; + var index = new Array(numVertices) + var lowValue = new Array(numVertices) + var active = new Array(numVertices) + var child = new Array(numVertices) + var scc = new Array(numVertices) + var sccLinks = new Array(numVertices) + + //Initialize tables + for (var i=0; i 0) { + v = T[T.length-1] + var e = adjList[v] + if (child[v] < e.length) { // If we're not done iterating over the children, first try finishing that. + for(var i=child[v]; i= 0) { + // Node v is not yet assigned an scc, but once it is that scc can apparently reach scc[u]. + sccLinks[v].push(scc[u]) + } + } + child[v] = i // Remember where we left off. + } else { // If we're done iterating over the children, check whether we have an scc. + if(lowValue[v] === index[v]) { // TODO: It /might/ be true that T is always a prefix of S (at this point!!!), and if so, this could be used here. + var component = [] + var links = [], linkCount = 0 + for(var i=S.length-1; i>=0; --i) { + var w = S[i] + active[w] = false + component.push(w) + links.push(sccLinks[w]) + linkCount += sccLinks[w].length + scc[w] = components.length + if(w === v) { + S.length = i + break + } + } + components.push(component) + var allLinks = new Array(linkCount) + for(var i=0; i", + "$@", + ]), +) + +# If the test passes, we probably emitted hello world +node_test( + name = "helloworld_test", + main = "helloworld.js", + visibility = ["//visibility:public"], +) + +# But let's make sure... +sh_test( + name = "generated_helloworld_test", + size = "small", + srcs = ["generated_helloworld_test.sh"], + data = [":helloworld.js"], +) diff --git a/tests/helloworld/gen/compiler.js b/tests/helloworld/gen/compiler.js new file mode 100644 index 0000000..b7f9992 --- /dev/null +++ b/tests/helloworld/gen/compiler.js @@ -0,0 +1,2 @@ +// This 'compiler' always emits a helloworld program +console.log(`console.log("Hello World");`); diff --git a/tests/helloworld/gen/generated_helloworld_test.sh b/tests/helloworld/gen/generated_helloworld_test.sh new file mode 100755 index 0000000..c38ac53 --- /dev/null +++ b/tests/helloworld/gen/generated_helloworld_test.sh @@ -0,0 +1,8 @@ +set -e + +# Check the content of generated program +if cat gen/helloworld.js | grep -q 'console.log("Hello World");'; then + echo "PASS" +else + exit 1 +fi diff --git a/tests/helloworld/gen/input.js b/tests/helloworld/gen/input.js new file mode 100644 index 0000000..88b8641 --- /dev/null +++ b/tests/helloworld/gen/input.js @@ -0,0 +1 @@ +// Nothing here, input is ignored by this compiler... diff --git a/tests/helloworld/helloworld_test.sh b/tests/helloworld/helloworld_test.sh index 774821b..9963a8b 100755 --- a/tests/helloworld/helloworld_test.sh +++ b/tests/helloworld/helloworld_test.sh @@ -1,6 +1,6 @@ set -e -if ./helloworld | grep -q 'Hello World!'; then +if ./helloworld_bin | grep -q 'Hello World!'; then echo "PASS" else exit 1 diff --git a/tests/helloworld/require_test.js b/tests/helloworld/require_test.js new file mode 100644 index 0000000..d31580d --- /dev/null +++ b/tests/helloworld/require_test.js @@ -0,0 +1,4 @@ +const hello = require('helloworld'); + +// We didn't fail on the require? Success! +process.exit(0) diff --git a/tests/lyrics/BUILD b/tests/lyrics/BUILD index 58d7805..e9b9347 100644 --- a/tests/lyrics/BUILD +++ b/tests/lyrics/BUILD @@ -7,7 +7,8 @@ node_binary( main = "lyrics.js", deps = [ "@yarn_modules//:underscore", - ] + ], + deploy = 'tar.gz', ) sh_test( diff --git a/tests/mocha/BUILD b/tests/mocha/BUILD index 99a8809..81ed36c 100644 --- a/tests/mocha/BUILD +++ b/tests/mocha/BUILD @@ -1,16 +1,11 @@ package(default_visibility = ["//visibility:public"]) -load("@org_pubref_rules_node//node:rules.bzl", "node_binary", "node_module", "mocha_test") +load("@org_pubref_rules_node//node:rules.bzl", "node_binary", "node_module", "node_modules", "mocha_test") mocha_test( name = "test", main = "test.js", + args = [ + ], + deps = ["//src/main/js:greeter"], ) - -# Can one test a bazel test? I'm getting 'fork: Resource temporarily unavailable' when I try to run this. -# sh_test( -# name = "test_test", -# size = "small", -# srcs = ["test.sh"], -# data = [":test"], -# ) diff --git a/tests/mocha/src/main/js/BUILD b/tests/mocha/src/main/js/BUILD new file mode 100644 index 0000000..006f6a3 --- /dev/null +++ b/tests/mocha/src/main/js/BUILD @@ -0,0 +1,9 @@ + +package(default_visibility = ["//visibility:public"]) + +load("@org_pubref_rules_node//node:rules.bzl", "node_module") + +node_module( + name = "greeter", + main = "greeter.js", +) diff --git a/tests/mocha/src/main/js/greeter.js b/tests/mocha/src/main/js/greeter.js new file mode 100644 index 0000000..c264637 --- /dev/null +++ b/tests/mocha/src/main/js/greeter.js @@ -0,0 +1,3 @@ +module.exports = function(name) { + return "Hello " + name + '!'; +} diff --git a/tests/mocha/tests/test.js b/tests/mocha/src/main/js/test.js similarity index 100% rename from tests/mocha/tests/test.js rename to tests/mocha/src/main/js/test.js diff --git a/tests/mocha/test.js b/tests/mocha/test.js index 3c07044..aacf131 100644 --- a/tests/mocha/test.js +++ b/tests/mocha/test.js @@ -1,3 +1,4 @@ +var greet = require('src/main/js/greeter'); var assert = require('assert'); describe('Array', function() { @@ -7,3 +8,9 @@ describe('Array', function() { }); }); }); + +describe('#greet()', function() { + it('should say hello', function() { + assert.equal('Hello World!', greet('World')); + }); +}); diff --git a/tests/mocha/tests/BUILD b/tests/mocha/tests/BUILD deleted file mode 100644 index 99a8809..0000000 --- a/tests/mocha/tests/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load("@org_pubref_rules_node//node:rules.bzl", "node_binary", "node_module", "mocha_test") - -mocha_test( - name = "test", - main = "test.js", -) - -# Can one test a bazel test? I'm getting 'fork: Resource temporarily unavailable' when I try to run this. -# sh_test( -# name = "test_test", -# size = "small", -# srcs = ["test.sh"], -# data = [":test"], -# ) diff --git a/tests/typescript/BUILD b/tests/typescript/BUILD index 05dea52..bb52c04 100644 --- a/tests/typescript/BUILD +++ b/tests/typescript/BUILD @@ -58,12 +58,3 @@ sh_test( data = [":ts_report"], ) - -# Note to self: genrule way to call typescript (too simple for real usage) -# genrule( -# name = "tsc", -# srcs = ["hello.ts"], -# outs = ["hello.js"], -# cmd = "external/yarn_modules/node_modules/.bin/tsc --outDir $(@D) $(SRCS)", -# tools = ["@yarn_modules//:tsc"], -# ) diff --git a/tests/typescript/ts_module.bzl b/tests/typescript/ts_module.bzl index 08b6eb4..92a118f 100644 --- a/tests/typescript/ts_module.bzl +++ b/tests/typescript/ts_module.bzl @@ -1,15 +1,12 @@ load("@org_pubref_rules_node//node:rules.bzl", "node_module") - -def _get_d_ts_files(list): - files = [] - for file in list: - if file.path.endswith(".d.ts"): - files.append(file) - return files - +# Note: this is not by any means production quality support for +# typescript. It's more of an experiment to exercise the +# behavior/utility of the node_module rule in various circumstances. def _build_node_module(ctx, compilation_dir, node_module): + """Copy the given node_module into the specified compilation dir""" + outputs = [] for src in node_module.sources: relpath = node_module.sourcemap[src.path] @@ -25,25 +22,59 @@ def _build_node_module(ctx, compilation_dir, node_module): return outputs -def _ts_module_impl(ctx): - node = ctx.executable._node - tsc = ctx.executable._tsc - tsconfig = ctx.file.tsconfig - inputs = [node, tsc] - if tsconfig: - inputs.append(tsconfig) +def _get_relative_path(file, rel): + """Get the path of a file relative to to rel. + + For example, if rel is the bazel-bin/foo/tsconfig.json and 'file' + is bazel-bin/foo/a/b/c.ts, return 'a/b/c.ts' + """ + + return file.path[len(rel.dirname) + 1:] + + +def _create_tsconfig(ctx, compilation_dir, srcs): + """Create the tsconfig.json file""" + + # The tsconfig file to be generated + tsconfig_file = ctx.new_file("%s/tsconfig.json" % compilation_dir) + + # The files that we want to compile + files = [_get_relative_path(file, tsconfig_file) for file in srcs] + + # Todo: add all the necessary attributes for the tsconfig + json = { + "name": ctx.label.name, + "files": files + } - compilation_dir = "package_" + ctx.label.name + ".tscompile" + # Generate a struct from the map + content = struct(**json) + + # Create a file action to generate it... + ctx.file_action( + output = tsconfig_file, + content = content.to_json(), + ) + + return tsconfig_file + + +def _ts_module_impl(ctx): + # Location where we'll copy all the needed files for the ts compilation. + compilation_dir = ctx.label.name + # Compilation inputs + inputs = [] + # Compilation outputs + outputs = [] - node_modules = [] # list of output files (building a custom node_modules tree for the compilation) + # list of module output files (building a custom node_modules tree + # for the compilation) + node_modules = [] for dep in ctx.attr.deps: node_modules += _build_node_module(ctx, compilation_dir, dep.node_module) - output_js_files = [] - output_js_map_files = [] - output_d_ts_files = [] - - srcs = [] + # Copy the source files into the compilation dir. + srcs = [] for src in ctx.files.srcs: copied_src = ctx.new_file("%s/%s" % (compilation_dir, src.short_path)) ctx.action( @@ -53,53 +84,52 @@ def _ts_module_impl(ctx): ) srcs.append(copied_src) + # Generate a *.js, *.d.ts, and *.js.map file foreach source *.ts file. for src in srcs: inputs.append(src) basefile = src.short_path[0:-len(src.extension) - 1] if ctx.label.package: basefile = ctx.label.package + "/" + basefile - js_out = ctx.new_file("%s.js" % basefile) - output_js_files.append(js_out) - d_ts_out = ctx.new_file("%s.d.ts" % basefile) - output_d_ts_files.append(d_ts_out) + js_out = ctx.new_file("%s.js" % (basefile)) + outputs.append(js_out) + d_ts_out = ctx.new_file("%s.d.ts" % (basefile)) + outputs.append(d_ts_out) if (ctx.attr.sourcemap): - js_map_out = ctx.new_file("%s.js.map" % basefile) - output_js_map_files.append(js_map_out) + js_map_out = ctx.new_file("%s.js.map" % (basefile)) + outputs.append(js_map_out) + + # Generate a tsconfig.json file + tsconfig = _create_tsconfig(ctx, compilation_dir, srcs) + # Setup args for tsc arguments = [ - tsc.path, "--moduleResolution", "node", "--declaration", + "--project", tsconfig.path, ] + ctx.attr.args if ctx.attr.sourcemap: arguments += ["--sourceMap"] - if tsconfig: - arguments += ["--project", tsconfig.path] - - for src in srcs: - arguments.append(src.path) - - outputs = output_js_files + output_d_ts_files + output_js_map_files - + # Run the compilation ctx.action( mnemonic = "TypescriptCompile", - inputs = inputs + node_modules, + inputs = inputs + node_modules + [tsconfig], outputs = outputs, - executable = node, + executable = ctx.executable._tsc, arguments = arguments, ) return struct( - files = depset(outputs), + files = depset(outputs + [tsconfig]), ts_module = struct( files = outputs, tsconfig = tsconfig, - srcs = ctx.files.srcs, + srcs = srcs, ) ) + _ts_module = rule( implementation = _ts_module_impl, attrs = { @@ -110,41 +140,31 @@ _ts_module = rule( "deps": attr.label_list( providers = ["node_module"], ), - "tsconfig": attr.label( - allow_files = FileType(["tsconfig.json"]), - single_file = True, - mandatory = False, - ), "sourcemap": attr.bool( default = True, ), "args": attr.string_list(), "_tsc": attr.label( - default = "@yarn_modules//:tsc_bin", - executable = True, - cfg = "host", - ), - "_node": attr.label( - default = Label("@node//:node"), - single_file = True, - allow_files = True, + default = "@yarn_modules//:typescript_tsc_bin", executable = True, cfg = "host", ), }, ) -def ts_module(name = None, srcs = [], tsconfig = None, deps = [], sourcemap = True, **kwargs): + +def ts_module(name = None, srcs = [], deps = [], **kwargs): _ts_module( - name = name + ".tsc", + name = name + "_ts", srcs = srcs, - tsconfig = tsconfig, - sourcemap = sourcemap, deps = deps, ) node_module( name = name, - srcs = [name + ".tsc"], + srcs = [name + "_ts"], deps = deps, + # It's a bit of a hack to use the 'flat' layout here, but it + # makes the import of generated modules simpler. + layout = "flat", **kwargs ) diff --git a/tests/webpack/BUILD b/tests/webpack/BUILD new file mode 100644 index 0000000..99fa7c7 --- /dev/null +++ b/tests/webpack/BUILD @@ -0,0 +1,40 @@ +package(default_visibility = ["//visibility:public"]) + +sh_test( + name = "webpack_bin_test", + size = "small", + srcs = ["webpack_bin_test.sh"], + data = ["@yarn_modules//:webpack_bin"], +) + +genrule( + name = "webpack_compile", + tools = [ + "@yarn_modules//:webpack_bin", + #"webpack.config.js", + ], + srcs = [ + "input.js", + ], + outs = [ + "bundle.js", + ], + cmd = " ".join([ + # Note: it is critical to use 'locations' and not 'location' + # here, presumably as the tool dependency includes many files + "$(locations @yarn_modules//:webpack_bin)", + "$(SRCS)", + "$@", + ]), +) + + +sh_test( + name = "webpack_compile_test", + size = "small", + srcs = ["webpack_compile_test.sh"], + data = [":webpack_compile"], # data dependency on the bundle.js + # file output implicitly invokes + # webback to generate it +) + diff --git a/tests/webpack/README.md b/tests/webpack/README.md new file mode 100644 index 0000000..60ebc12 --- /dev/null +++ b/tests/webpack/README.md @@ -0,0 +1,25 @@ +# Webpack Example + +This folder demonstrates that the generated `BUILD` file for +`@yarn_modules` contains a `node_binary` rule for the webpack module +executable. It tests that the target is callable and that the help +message is output. + +Webpack is a fairly complex dependency that contains a cycle in +`es5-ext`, `es6-iterator`, `es6-symbol`, `d` dependency cluster. It +breaks this strongly connected component into a separate pseudo +`node_module`. + +```sh +# Should be able to run webpack directly +$ bazel build @yarn_modules//:webback_bin -- --help + +# Should be able to invoke webpack as standalone script +$ ./bazel-bin/external/yarn_modules/webpack_bin --help + +# Should be able to execute webpack as part of a genrule +$ bazel build :webpack_compile + +# Should be able to invoke another (contrived) genrule +$ bazel build :compile +``` diff --git a/tests/webpack/WORKSPACE b/tests/webpack/WORKSPACE new file mode 100644 index 0000000..dc4eb9c --- /dev/null +++ b/tests/webpack/WORKSPACE @@ -0,0 +1,34 @@ +local_repository( + name = "org_pubref_rules_node", + path = "../..", +) + +load("@org_pubref_rules_node//node:rules.bzl", "node_repositories", "yarn_modules") + +node_repositories() + +yarn_modules( + name = "yarn_modules", + deps = { + "webpack": "3.4.0", + }, + resolutions = { + "acorn": "5.1.2", + "ajv": "5.2.3", + "ansi-regex": "3.0.0", + "assert-plus": "1.0.0", + "camelcase": "4.1.0", + "cliui": "3.2.0", + "hash-base": "3.0.4", + "inherits": "2.0.3", + "is-fullwidth-code-point": "2.0.0", + "is-number": "3.0.0", + "kind-of": "4.0.0", + "minimist": "1.2.0", + "punycode": "1.4.1", + "string-width": "2.1.1", + "string_decoder": "1.0.3", + "strip-ansi": "4.0.0", + "yargs": "8.0.2", + } +) diff --git a/tests/webpack/input.js b/tests/webpack/input.js new file mode 100644 index 0000000..a420803 --- /dev/null +++ b/tests/webpack/input.js @@ -0,0 +1 @@ +console.log('Hello World!'); diff --git a/tests/webpack/webpack_bin_test.sh b/tests/webpack/webpack_bin_test.sh new file mode 100755 index 0000000..d6dfb3d --- /dev/null +++ b/tests/webpack/webpack_bin_test.sh @@ -0,0 +1,8 @@ +set -e + + +if (./external/yarn_modules/webpack_bin --help &) | grep -q 'webpack 3.4.0'; then + echo "PASS" +else + exit 1 +fi diff --git a/tests/webpack/webpack_compile_test.sh b/tests/webpack/webpack_compile_test.sh new file mode 100755 index 0000000..32446ec --- /dev/null +++ b/tests/webpack/webpack_compile_test.sh @@ -0,0 +1,8 @@ +set -e + +# Check for webpack signature in the compiled file (present on first line) +if (cat bundle.js &) | grep -q '// webpackBootstrap'; then + echo "PASS" +else + exit 1 +fi