Merge pull request #119 from koloml/feature/amd-loaded-content-scripts

Reworked build step for content scripts & styles to use AMD as module system

.vite/lib/content-scripts.js

@@ -1,5 +1,5 @@
-import {build} from "vite";
-import {createHash} from "crypto";
+import { build } from "vite";
+import { createHash } from "crypto";
 import path from "path";
 import fs from "fs";
 
@@ -56,69 +56,167 @@ function makeAliases(rootDir) {
 }
 
 /**
- * Build the selected script separately.
- * @param {AssetBuildOptions} buildOptions Building options for the script.
- * @return {Promise<string>} Result file path.
+ * @param {import('rollup').OutputChunk} chunk
+ * @param {import('rollup').OutputBundle} bundle
+ * @param {Set<import('rollup').OutputChunk>} processedChunks
+ * @return string[]
  */
-export async function buildScript(buildOptions) {
-  const outputBaseName = createOutputBaseName(buildOptions.input);
+function collectChunkDependencies(chunk, bundle, processedChunks = new Set()) {
+  if (processedChunks.has(chunk) || !chunk.imports) {
+    return [];
+  }
 
+  processedChunks.add(chunk);
+
+  return chunk.imports.concat(
+    chunk.imports
+      .map(importedChunkName => {
+        const module = bundle[importedChunkName];
+
+        if (module.type === 'chunk') {
+          return collectChunkDependencies(module, bundle, processedChunks);
+        }
+
+        return [];
+      })
+      .flat()
+  );
+}
+
+/**
+ * @param {(fileName: string, dependencies: string[]) => void} onDependencyResolvedCallback
+ * @returns {import('vite').Plugin}
+ */
+function collectDependenciesForManifestBuilding(onDependencyResolvedCallback) {
+  return {
+    name: 'extract-dependencies-for-content-scripts',
+    enforce: "post",
+    /**
+     * @param {any} options
+     * @param {import('rollup').OutputBundle} bundle
+     */
+    writeBundle(options, bundle) {
+      Object.keys(bundle).forEach(fileName => {
+        const chunk = bundle[fileName];
+
+        if (chunk.type !== "chunk" || !chunk.facadeModuleId) {
+          return;
+        }
+
+        const dependencies = Array.from(
+          new Set(
+            collectChunkDependencies(chunk, bundle)
+          )
+        );
+
+        onDependencyResolvedCallback(fileName, dependencies);
+      });
+    }
+  }
+}
+
+/**
+ * Second revision of the building logic for the content scripts. This method tries to address duplication of
+ * dependencies generated with the previous method, where every single content script was built separately.
+ * @param {BatchBuildOptions} buildOptions
+ * @returns {Promise<Map<string, string[]>>}
+ */
+export async function buildScriptsAndStyles(buildOptions) {
+  /** @type {Map<string, string[]>} */
+  const pathsReplacement = new Map();
+  /** @type {Map<string, string[]>} */
+  const pathsReplacementByOutputPath = new Map();
+
+  const amdScriptsInput = {};
+  const libsAndStylesInput = {};
+
+  for (const inputPath of buildOptions.inputs) {
+    let outputExtension = path.extname(inputPath);
+
+    if (outputExtension === '.scss') {
+      outputExtension = '.css';
+    }
+
+    if (outputExtension === '.ts') {
+      outputExtension = '.js';
+    }
+
+    const outputPath = createOutputBaseName(inputPath);
+    const replacementsArray = [`${outputPath}${outputExtension}`];
+
+    pathsReplacement.set(inputPath, replacementsArray);
+
+    if (outputExtension === '.css' || inputPath.includes('/deps/')) {
+      libsAndStylesInput[outputPath] = inputPath;
+      continue;
+    }
+
+    pathsReplacementByOutputPath.set(outputPath + '.js', replacementsArray);
+
+    amdScriptsInput[outputPath] = inputPath;
+  }
+
+  const aliasesSettings = makeAliases(buildOptions.rootDir);
+
+  // Building all scripts together with AMD loader in mind
   await build({
     configFile: false,
     publicDir: false,
     build: {
       rollupOptions: {
-        input: {
-          [outputBaseName]: buildOptions.input
-        },
+        input: amdScriptsInput,
         output: {
           dir: buildOptions.outputDir,
-          entryFileNames: '[name].js'
+          entryFileNames: '[name].js',
+          chunkFileNames: 'chunks/[name]-[hash].js',
+          // ManifestV3 doesn't allow to use modern ES modules syntax, so we build all content scripts as AMD modules.
+          format: "amd",
+          inlineDynamicImports: false,
+          amd: {
+            // amd-lite requires names even for the entry-point scripts, so we should make sure to add those.
+            autoId: true,
+          }
         }
       },
       emptyOutDir: false,
     },
     resolve: {
-      alias: makeAliases(buildOptions.rootDir)
+      alias: aliasesSettings,
     },
     plugins: [
-      wrapScriptIntoIIFE()
+      wrapScriptIntoIIFE(),
+      collectDependenciesForManifestBuilding((fileName, dependencies) => {
+        pathsReplacementByOutputPath
+          .get(fileName)
+          ?.push(...dependencies);
+      }),
     ]
   });
 
-  return path.resolve(buildOptions.outputDir, `${outputBaseName}.js`);
-}
-
-/**
- * Build the selected stylesheet.
- * @param {AssetBuildOptions} buildOptions Build options for the stylesheet.
- * @return {Promise<string>} Result file path.
- */
-export async function buildStyle(buildOptions) {
-  const outputBaseName = createOutputBaseName(buildOptions.input);
-
+  // Build styles separately because AMD converts styles to JS files.
   await build({
     configFile: false,
     publicDir: false,
     build: {
       rollupOptions: {
-        input: {
-          [outputBaseName]: buildOptions.input
-        },
+        input: libsAndStylesInput,
         output: {
           dir: buildOptions.outputDir,
           entryFileNames: '[name].js',
           assetFileNames: '[name].[ext]',
         }
       },
-      emptyOutDir: false,
+      emptyOutDir: false
     },
     resolve: {
-      alias: makeAliases(buildOptions.rootDir)
-    }
+      alias: aliasesSettings,
+    },
+    plugins: [
+      wrapScriptIntoIIFE(),
+    ]
   });
 
-  return path.resolve(buildOptions.outputDir, `${outputBaseName}.css`);
+  return pathsReplacement;
 }
 
 /**
@@ -127,3 +225,11 @@ export async function buildStyle(buildOptions) {
  * @property {string} outputDir Destination folder for the script.
  * @property {string} rootDir Root directory of the repository.
  */
+
+/**
+ * @typedef {Object} BatchBuildOptions
+ * @property {Set<string>} inputs Set of all scripts and styles to build.
+ * @property {string} outputDir Destination folder for the assets.
+ * @property {string} rootDir Root directory of the repository.
+ * @property {(fileName: string, dependencies: string[]) => void} onDependenciesResolved Callback for dependencies.
+ */

.vite/lib/manifest.js

@@ -17,6 +17,38 @@ class ManifestProcessor {
     this.#manifestObject = parsedManifest;
   }
 
+  /**
+   * Collect all the content scripts & stylesheets for single build action.
+   *
+   * @returns {Set<string>}
+   */
+  collectContentScripts() {
+    const contentScripts = this.#manifestObject.content_scripts;
+
+    if (!contentScripts) {
+      console.info('No content scripts to collect.');
+      return new Set();
+    }
+
+    const entryPoints = new Set();
+
+    for (let entry of contentScripts) {
+      if (entry.js) {
+        for (let jsPath of entry.js) {
+          entryPoints.add(jsPath);
+        }
+      }
+
+      if (entry.css) {
+        for (let cssPath of entry.css) {
+          entryPoints.add(cssPath);
+        }
+      }
+    }
+
+    return entryPoints;
+  }
+
   /**
    * Map over every content script defined in the manifest. If no content scripts defined, no calls will be made to the
    * callback.

.vite/pack-extension.js

@@ -1,8 +1,8 @@
-import {loadManifest} from "./lib/manifest.js";
+import { loadManifest } from "./lib/manifest.js";
 import path from "path";
-import {buildScript, buildStyle} from "./lib/content-scripts.js";
-import {normalizePath} from "vite";
-import {extractInlineScriptsFromIndex} from "./lib/index-file.js";
+import { buildScriptsAndStyles } from "./lib/content-scripts.js";
+import { extractInlineScriptsFromIndex } from "./lib/index-file.js";
+import { normalizePath } from "vite";
 
 /**
  * Build addition assets required for the extension and pack it into the directory.
@@ -11,45 +11,61 @@ import {extractInlineScriptsFromIndex} from "./lib/index-file.js";
 export async function packExtension(settings) {
   const manifest = loadManifest(path.resolve(settings.rootDir, 'manifest.json'));
 
-  // Since we CAN'T really build all scripts and stylesheets in a single build entry, we will run build for every single
-  // one of them in a row. This way, no chunks will be generated. Thanks, ManifestV3!
-  await manifest.mapContentScripts(async (entry) => {
-    if (entry.js) {
-      for (let scriptIndex = 0; scriptIndex < entry.js.length; scriptIndex++) {
-        const builtScriptFilePath = await buildScript({
-          input: path.resolve(settings.rootDir, entry.js[scriptIndex]),
-          outputDir: settings.contentScriptsDir,
-          rootDir: settings.rootDir,
-        });
+  const replacementMapping = await buildScriptsAndStyles({
+    inputs: manifest.collectContentScripts(),
+    outputDir: settings.contentScriptsDir,
+    rootDir: settings.rootDir,
+  });
 
-        entry.js[scriptIndex] = normalizePath(
-          path.relative(
-            settings.exportDir,
-            builtScriptFilePath
+  await manifest.mapContentScripts(async entry => {
+    if (entry.js) {
+      entry.js = entry.js
+        .map(jsSourcePath => {
+          if (!replacementMapping.has(jsSourcePath)) {
+            return [];
+          }
+
+          return replacementMapping.get(jsSourcePath);
+        })
+        .flat(1)
+        .map(pathName => {
+          return normalizePath(
+            path.relative(
+              settings.exportDir,
+              path.join(
+                settings.contentScriptsDir,
+                pathName
+              )
+            )
           )
-        );
-      }
+        });
     }
 
     if (entry.css) {
-      for (let styleIndex = 0; styleIndex < entry.css.length; styleIndex++) {
-        const builtStylesheetFilePath = await buildStyle({
-          input: path.resolve(settings.rootDir, entry.css[styleIndex]),
-          outputDir: settings.contentScriptsDir,
-          rootDir: settings.rootDir
-        });
+      entry.css = entry.css
+        .map(jsSourcePath => {
+          if (!replacementMapping.has(jsSourcePath)) {
+            return [];
+          }
 
-        entry.css[styleIndex] = normalizePath(
-          path.relative(
-            settings.exportDir,
-            builtStylesheetFilePath
+          return replacementMapping.get(jsSourcePath);
+        })
+        .flat(1)
+        .map(pathName => {
+          return normalizePath(
+            path.relative(
+              settings.exportDir,
+              path.join(
+                settings.contentScriptsDir,
+                pathName
+              )
+            )
           )
-        );
-      }
+        })
     }
 
     return entry;
-  });
+  })
 
   manifest.passVersionFromPackage(path.resolve(settings.rootDir, 'package.json'));
   manifest.saveTo(path.resolve(settings.exportDir, 'manifest.json'));

manifest.json

@@ -18,6 +18,14 @@
         "*://*.furbooru.org/"
     ],
     "content_scripts": [
+        {
+            "matches": [
+                "*://*.furbooru.org/*"
+            ],
+            "js": [
+                "src/content/deps/amd.ts"
+            ]
+        },
         {
             "matches": [
                 "*://*.furbooru.org/",

package-lock.json

@@ -9,6 +9,7 @@
             "version": "0.4.4",
             "dependencies": {
                 "@fortawesome/fontawesome-free": "^6.7.2",
+                "amd-lite": "^1.0.1",
                 "lz-string": "^1.5.0"
             },
             "devDependencies": {
@@ -1388,6 +1389,12 @@
                 "node": ">= 14"
             }
         },
+        "node_modules/amd-lite": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/amd-lite/-/amd-lite-1.0.1.tgz",
+            "integrity": "sha512-2EqBXjF5YjgCHeb4hfhUx3iIiN/vmuXjXdT1QVDXylwH5c2oqEvGPDmyVo9yGRGEQlGRdwa9gJ68/m32yUnriw==",
+            "license": "MIT"
+        },
         "node_modules/ansi-regex": {
             "version": "6.1.0",
             "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz",

package.json

@@ -31,6 +31,7 @@
     "type": "module",
     "dependencies": {
         "@fortawesome/fontawesome-free": "^6.7.2",
+        "amd-lite": "^1.0.1",
         "lz-string": "^1.5.0"
     }
 }

src/content/deps/amd.ts

@@ -0,0 +1,22 @@
+import { amdLite } from "amd-lite";
+
+const originalDefine = amdLite.define;
+
+amdLite.define = (name, dependencies, originalCallback) => {
+  return originalDefine(name, dependencies, function () {
+    const callbackResult = originalCallback(...arguments);
+
+    // Workaround for the entry script not returning anything causing AMD-Lite to send warning about modules not
+    // being loaded/not existing.
+    return typeof callbackResult !== 'undefined' ? callbackResult : {};
+  })
+}
+
+amdLite.init({
+  publicScope: window
+});
+
+// We don't have anything asynchronous, so it's safe to execute everything on the next frame.
+requestAnimationFrame(() => {
+  amdLite.resolveDependencies(Object.keys(amdLite.waitingModules))
+});

src/types/amd-lite.d.ts

@@ -0,0 +1,23 @@
+// Types for the small untyped AMD loader. These types do not cover all the functions available in the package, only
+// parts required for content scripts in extension to work.
+declare module 'amd-lite' {
+  interface AMDLiteInitOptions {
+    publicScope: any;
+    verbosity: number;
+  }
+
+  interface AMDLite {
+    waitingModules: Record<string, any>;
+    readyModules: Record<string, any>;
+
+    init(options: Partial<AMDLiteInitOptions>): void;
+
+    define(name: string, dependencies: string[], callback: function): void;
+
+    resolveDependency(dependencyPath: string);
+
+    resolveDependencies(dependencyNames: string[], from?: string);
+  }
+
+  export const amdLite: AMDLite;
+}