diff --git a/etc/api/angular_devkit/build_optimizer/src/_golden-api.d.ts b/etc/api/angular_devkit/build_optimizer/src/_golden-api.d.ts
index c545afc96c41..83e1865b8cba 100644
--- a/etc/api/angular_devkit/build_optimizer/src/_golden-api.d.ts
+++ b/etc/api/angular_devkit/build_optimizer/src/_golden-api.d.ts
@@ -1,5 +1,11 @@
 export declare function buildOptimizer(options: BuildOptimizerOptions): TransformJavascriptOutput;
 
+export declare const buildOptimizerLoaderPath: string;
+
+export declare class BuildOptimizerWebpackPlugin {
+    apply(compiler: Compiler): void;
+}
+
 export default function buildOptimizerLoader(this: webpack.loader.LoaderContext, content: string, previousSourceMap: RawSourceMap): void;
 
 export declare function getFoldFileTransformer(program: ts.Program): ts.TransformerFactory<ts.SourceFile>;
diff --git a/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts b/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts
index a553b2d98508..e2677039b0f7 100644
--- a/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts
+++ b/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/common.ts
@@ -5,6 +5,10 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
+import {
+  BuildOptimizerWebpackPlugin,
+  buildOptimizerLoaderPath,
+} from '@angular-devkit/build-optimizer';
 import { tags } from '@angular-devkit/core';
 import * as CopyWebpackPlugin from 'copy-webpack-plugin';
 import * as path from 'path';
@@ -34,9 +38,6 @@ const TerserPlugin = require('terser-webpack-plugin');
 
 // tslint:disable-next-line:no-any
 const g: any = typeof global !== 'undefined' ? global : {};
-export const buildOptimizerLoader: string = g['_DevKitIsLocal']
-  ? require.resolve('@angular-devkit/build-optimizer/src/build-optimizer/webpack-loader')
-  : '@angular-devkit/build-optimizer/webpack-loader';
 
 // tslint:disable-next-line:no-big-function
 export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
@@ -251,10 +252,11 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
 
   let buildOptimizerUseRule;
   if (buildOptions.buildOptimizer) {
+    extraPlugins.push(new BuildOptimizerWebpackPlugin());
     buildOptimizerUseRule = {
       use: [
         {
-          loader: buildOptimizerLoader,
+          loader: buildOptimizerLoaderPath,
           options: { sourceMap: scriptsSourceMap },
         },
       ],
diff --git a/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/typescript.ts b/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/typescript.ts
index 9660f0fe662a..eef882522e00 100644
--- a/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/typescript.ts
+++ b/packages/angular_devkit/build_angular/src/angular-cli-files/models/webpack-configs/typescript.ts
@@ -7,6 +7,7 @@
  */
 // tslint:disable
 // TODO: cleanup this file, it's copied as is from Angular CLI.
+import { buildOptimizerLoaderPath } from '@angular-devkit/build-optimizer';
 import * as path from 'path';
 import {
   AngularCompilerPlugin,
@@ -14,7 +15,6 @@ import {
   NgToolsLoader,
   PLATFORM
 } from '@ngtools/webpack';
-import { buildOptimizerLoader } from './common';
 import { WebpackConfigOptions, BuildOptions } from '../build-options';
 
 function _pluginOptionsOverrides(
@@ -113,7 +113,7 @@ export function getAotConfig(wco: WebpackConfigOptions, i18nExtract = false) {
   const loaders: any[] = [NgToolsLoader];
   if (buildOptions.buildOptimizer) {
     loaders.unshift({
-      loader: buildOptimizerLoader,
+      loader: buildOptimizerLoaderPath,
       options: { sourceMap: buildOptions.sourceMap.scripts }
     });
   }
diff --git a/packages/angular_devkit/build_optimizer/README.md b/packages/angular_devkit/build_optimizer/README.md
index 0a7b3236badb..6b97c1b3588d 100644
--- a/packages/angular_devkit/build_optimizer/README.md
+++ b/packages/angular_devkit/build_optimizer/README.md
@@ -186,7 +186,12 @@ export interface BuildOptimizerOptions {
 ## Webpack loader usage:
 
 ```typescript
+import { BuildOptimizerWebpackPlugin } from '@angular-devkit/build-optimizer';
+
 module.exports = {
+  plugins: [
+    new BuildOptimizerWebpackPlugin(),
+  ]
   module: {
     rules: [
       {
diff --git a/packages/angular_devkit/build_optimizer/src/build-optimizer/webpack-loader.ts b/packages/angular_devkit/build_optimizer/src/build-optimizer/webpack-loader.ts
index e05666f8b596..ec4c138a7900 100644
--- a/packages/angular_devkit/build_optimizer/src/build-optimizer/webpack-loader.ts
+++ b/packages/angular_devkit/build_optimizer/src/build-optimizer/webpack-loader.ts
@@ -17,9 +17,33 @@ interface BuildOptimizerLoaderOptions {
   sourceMap: boolean;
 }
 
+export const buildOptimizerLoaderPath = __filename;
+
+const alwaysProcess = (path: string) =>
+  // Always process TS files.
+  path.endsWith('.ts') || path.endsWith('.tsx')
+  // Always process factory files.
+  || path.endsWith('.ngfactory.js') || path.endsWith('.ngstyle.js');
+
 export default function buildOptimizerLoader
   (this: webpack.loader.LoaderContext, content: string, previousSourceMap: RawSourceMap) {
   this.cacheable();
+
+  const skipBuildOptimizer = this._module
+    && this._module.factoryMeta
+    && this._module.factoryMeta.skipBuildOptimizer;
+
+  if (!alwaysProcess(this.resourcePath) && skipBuildOptimizer) {
+    // Skip loading processing this file with Build Optimizer if we determined in
+    // BuildOptimizerWebpackPlugin that we shouldn't.
+
+    // Webpack typings for previousSourceMap are wrong, they are JSON objects and not strings.
+    // tslint:disable-next-line:no-any
+    this.callback(null, content, previousSourceMap as any);
+
+    return;
+  }
+
   const options: BuildOptimizerLoaderOptions = loaderUtils.getOptions(this) || {};
 
   // Make up names of the intermediate files so we can chain the sourcemaps.
@@ -33,12 +57,11 @@ export default function buildOptimizerLoader
     outputFilePath,
     emitSourceMap: options.sourceMap,
     isSideEffectFree: this._module
-                      && this._module.factoryMeta
-                      && this._module.factoryMeta.sideEffectFree,
+      && this._module.factoryMeta
+      && this._module.factoryMeta.sideEffectFree,
   });
 
   if (boOutput.emitSkipped || boOutput.content === null) {
-    // Webpack typings for previousSourceMap are wrong, they are JSON objects and not strings.
     // tslint:disable-next-line:no-any
     this.callback(null, content, previousSourceMap as any);
 
diff --git a/packages/angular_devkit/build_optimizer/src/build-optimizer/webpack-plugin.ts b/packages/angular_devkit/build_optimizer/src/build-optimizer/webpack-plugin.ts
new file mode 100644
index 000000000000..b31f0ecbb6cf
--- /dev/null
+++ b/packages/angular_devkit/build_optimizer/src/build-optimizer/webpack-plugin.ts
@@ -0,0 +1,33 @@
+
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { Compiler } from 'webpack';  // tslint:disable-line:no-implicit-dependencies
+
+export class BuildOptimizerWebpackPlugin {
+  apply(compiler: Compiler) {
+    compiler.hooks.normalModuleFactory.tap("BuildOptimizerWebpackPlugin", nmf => {
+      nmf.hooks.module.tap("BuildOptimizerWebpackPlugin", (module, data) => {
+        const resolveData = data.resourceResolveData;
+        if (resolveData && resolveData.descriptionFileData) {
+          // Only TS packages should use Build Optimizer.
+          const typings = resolveData.descriptionFileData.typings;
+          // Notes:
+          // - a TS package might not have defined typings but still use .d.ts files next to their
+          // .js files. We don't cover that case because the Angular Package Format (APF) calls for
+          // using the Typings field and Build Optimizer is geared towards APF. Maybe we could
+          // provide configuration options to the plugin to cover that case if there's demand.
+          // - a JS-only package that also happens to provides typings will also be flagged by this
+          // check. Not sure there's a good way to skip those.
+          module.factoryMeta.skipBuildOptimizer = !typings;
+        }
+
+        return module;
+      });
+    });
+  }
+}
diff --git a/packages/angular_devkit/build_optimizer/src/index.ts b/packages/angular_devkit/build_optimizer/src/index.ts
index 78ea03851fa8..96b201de9f66 100644
--- a/packages/angular_devkit/build_optimizer/src/index.ts
+++ b/packages/angular_devkit/build_optimizer/src/index.ts
@@ -5,7 +5,11 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-export { default as buildOptimizerLoader } from './build-optimizer/webpack-loader';
+export {
+  default as buildOptimizerLoader,
+  buildOptimizerLoaderPath,
+} from './build-optimizer/webpack-loader';
+export { BuildOptimizerWebpackPlugin } from './build-optimizer/webpack-plugin';
 export { buildOptimizer } from './build-optimizer/build-optimizer';
 
 export { transformJavascript } from './helpers/transform-javascript';