Skip to content

Commit b05a325

Browse files
alexander-akaitalexander-akait
authored
fix: avoid cache invalidation using ProgressPlugin 
2 parents ce179d7 + 13dc07e commit b05a325

File tree

19 files changed

+241
-11
lines changed

19 files changed

+241
-11
lines changed

‎lib/ProgressPlugin.js‎

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,18 @@ const { contextify } = require("./util/identifier");
1515
/** @typedef {import("../declarations/plugins/ProgressPlugin").HandlerFunction} HandlerFunction */
1616
/** @typedef {import("../declarations/plugins/ProgressPlugin").ProgressPluginArgument} ProgressPluginArgument */
1717
/** @typedef {import("../declarations/plugins/ProgressPlugin").ProgressPluginOptions} ProgressPluginOptions */
18+
/** @typedef {import("./Compilation").FactorizeModuleOptions} FactorizeModuleOptions */
1819
/** @typedef {import("./Dependency")} Dependency */
1920
/** @typedef {import("./Entrypoint").EntryOptions} EntryOptions */
2021
/** @typedef {import("./Module")} Module */
22+
/** @typedef {import("./ModuleFactory").ModuleFactoryResult} ModuleFactoryResult */
2123
/** @typedef {import("./logging/Logger").Logger} Logger */
2224

25+
/**
26+
* @template T, K, R
27+
* @typedef {import("./util/AsyncQueue")<T, K, R>} AsyncQueue
28+
*/
29+
2330
/**
2431
* @typedef {object} CountsData
2532
* @property {number} modulesCount modules count
@@ -121,6 +128,8 @@ const createDefaultHandler = (profile, logger) => {
121128
return defaultHandler;
122129
};
123130

131+
const SKIPPED_QUEUE_CONTEXTS = ["import-module", "load-module"];
132+
124133
/**
125134
* @callback ReportProgress
126135
* @param {number} p percentage
@@ -217,7 +226,9 @@ class ProgressPlugin {
217226
let lastDependenciesCount = 0;
218227
let lastEntriesCount = 0;
219228
let modulesCount = 0;
229+
let skippedModulesCount = 0;
220230
let dependenciesCount = 0;
231+
let skippedDependenciesCount = 0;
221232
let entriesCount = 1;
222233
let doneModules = 0;
223234
let doneDependencies = 0;
@@ -298,7 +309,15 @@ class ProgressPlugin {
298309
lastUpdate = Date.now();
299310
};
300311

301-
const factorizeAdd = () => {
312+
/**
313+
* @template T
314+
* @param {AsyncQueue<FactorizeModuleOptions, string, Module | ModuleFactoryResult>} factorizeQueue async queue
315+
* @param {T} _item item
316+
*/
317+
const factorizeAdd = (factorizeQueue, _item) => {
318+
if (SKIPPED_QUEUE_CONTEXTS.includes(factorizeQueue.getContext())) {
319+
skippedDependenciesCount++;
320+
}
302321
dependenciesCount++;
303322
if (dependenciesCount < 50 || dependenciesCount % 100 === 0)
304323
updateThrottled();
@@ -310,7 +329,15 @@ class ProgressPlugin {
310329
updateThrottled();
311330
};
312331

313-
const moduleAdd = () => {
332+
/**
333+
* @template T
334+
* @param {AsyncQueue<Module, string, Module>} addModuleQueue async queue
335+
* @param {T} _item item
336+
*/
337+
const moduleAdd = (addModuleQueue, _item) => {
338+
if (SKIPPED_QUEUE_CONTEXTS.includes(addModuleQueue.getContext())) {
339+
skippedModulesCount++;
340+
}
314341
modulesCount++;
315342
if (modulesCount < 50 || modulesCount % 100 === 0) updateThrottled();
316343
};
@@ -397,12 +424,19 @@ class ProgressPlugin {
397424
if (compilation.compiler.isChild()) return Promise.resolve();
398425
return /** @type {Promise<CountsData>} */ (cacheGetPromise).then(
399426
async oldData => {
427+
const realModulesCount = modulesCount - skippedModulesCount;
428+
const realDependenciesCount =
429+
dependenciesCount - skippedDependenciesCount;
430+
400431
if (
401432
!oldData ||
402-
oldData.modulesCount !== modulesCount ||
403-
oldData.dependenciesCount !== dependenciesCount
433+
oldData.modulesCount !== realModulesCount ||
434+
oldData.dependenciesCount !== realDependenciesCount
404435
) {
405-
await cache.storePromise({ modulesCount, dependenciesCount });
436+
await cache.storePromise({
437+
modulesCount: realModulesCount,
438+
dependenciesCount: realDependenciesCount
439+
});
406440
}
407441
}
408442
);
@@ -413,19 +447,25 @@ class ProgressPlugin {
413447
lastModulesCount = modulesCount;
414448
lastEntriesCount = entriesCount;
415449
lastDependenciesCount = dependenciesCount;
416-
modulesCount = dependenciesCount = entriesCount = 0;
450+
modulesCount =
451+
skippedModulesCount =
452+
dependenciesCount =
453+
skippedDependenciesCount =
454+
entriesCount =
455+
0;
417456
doneModules = doneDependencies = doneEntries = 0;
418457

419-
compilation.factorizeQueue.hooks.added.tap(
420-
"ProgressPlugin",
421-
factorizeAdd
458+
compilation.factorizeQueue.hooks.added.tap("ProgressPlugin", item =>
459+
factorizeAdd(compilation.factorizeQueue, item)
422460
);
423461
compilation.factorizeQueue.hooks.result.tap(
424462
"ProgressPlugin",
425463
factorizeDone
426464
);
427465

428-
compilation.addModuleQueue.hooks.added.tap("ProgressPlugin", moduleAdd);
466+
compilation.addModuleQueue.hooks.added.tap("ProgressPlugin", item =>
467+
moduleAdd(compilation.addModuleQueue, item)
468+
);
429469
compilation.processDependenciesQueue.hooks.result.tap(
430470
"ProgressPlugin",
431471
moduleDone

‎lib/dependencies/LoaderPlugin.js‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ class LoaderPlugin {
7676
)
7777
);
7878
}
79+
const oldFactorizeQueueContext =
80+
compilation.factorizeQueue.getContext();
81+
compilation.factorizeQueue.setContext("load-module");
82+
const oldAddModuleQueueContext =
83+
compilation.addModuleQueue.getContext();
84+
compilation.addModuleQueue.setContext("load-module");
7985
compilation.buildQueue.increaseParallelism();
8086
compilation.handleModuleCreation(
8187
{
@@ -88,6 +94,8 @@ class LoaderPlugin {
8894
recursive: false
8995
},
9096
err => {
97+
compilation.factorizeQueue.setContext(oldFactorizeQueueContext);
98+
compilation.addModuleQueue.setContext(oldAddModuleQueueContext);
9199
compilation.buildQueue.decreaseParallelism();
92100
if (err) {
93101
return callback(err);
@@ -173,6 +181,13 @@ class LoaderPlugin {
173181
)
174182
);
175183
}
184+
185+
const oldFactorizeQueueContext =
186+
compilation.factorizeQueue.getContext();
187+
compilation.factorizeQueue.setContext("import-module");
188+
const oldAddModuleQueueContext =
189+
compilation.addModuleQueue.getContext();
190+
compilation.addModuleQueue.setContext("import-module");
176191
compilation.buildQueue.increaseParallelism();
177192
compilation.handleModuleCreation(
178193
{
@@ -189,6 +204,8 @@ class LoaderPlugin {
189204
checkCycle: true
190205
},
191206
err => {
207+
compilation.factorizeQueue.setContext(oldFactorizeQueueContext);
208+
compilation.addModuleQueue.setContext(oldAddModuleQueueContext);
192209
compilation.buildQueue.decreaseParallelism();
193210
if (err) {
194211
return callback(err);

‎lib/util/AsyncQueue.js‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,14 @@ class AsyncQueue {
6868
* @param {object} options options object
6969
* @param {string=} options.name name of the queue
7070
* @param {number=} options.parallelism how many items should be processed at once
71+
* @param {string=} options.context context of execution
7172
* @param {AsyncQueue<any, any, any>=} options.parent parent queue, which will have priority over this queue and with shared parallelism
7273
* @param {getKey<T, K>=} options.getKey extract key from item
7374
* @param {Processor<T, R>} options.processor async function to process items
7475
*/
75-
constructor({ name, parallelism, parent, processor, getKey }) {
76+
constructor({ name, context, parallelism, parent, processor, getKey }) {
7677
this._name = name;
78+
this._context = context || "normal";
7779
this._parallelism = parallelism || 1;
7880
this._processor = processor;
7981
this._getKey =
@@ -115,6 +117,20 @@ class AsyncQueue {
115117
this._ensureProcessing = this._ensureProcessing.bind(this);
116118
}
117119

120+
/**
121+
* @returns {string} context of execution
122+
*/
123+
getContext() {
124+
return this._context;
125+
}
126+
127+
/**
128+
* @param {string} value context of execution
129+
*/
130+
setContext(value) {
131+
this._context = value;
132+
}
133+
118134
/**
119135
* @param {T} item an item
120136
* @param {Callback<R>} callback callback function

‎test/watchCases/cache/loader-import-module-progress/0/a.generate-json.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
export const value = 42;
2+
export * from "./imported.js";
3+
export { default as nested } from "./b.generate-json.js";
4+
export const random = Math.random();

‎test/watchCases/cache/loader-import-module-progress/0/b.generate-json.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export const value = 42;
2+
export * from "./imported.js";

‎test/watchCases/cache/loader-import-module-progress/0/imported.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export const a = "a";
2+
export const b = "b";

‎test/watchCases/cache/loader-import-module-progress/0/index.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import a from "./a.generate-json.js";
2+
import { value as unrelated } from "./unrelated";
3+
4+
it("should have to correct values and validate on change", () => {
5+
const step = +WATCH_STEP;
6+
expect(a.value).toBe(42);
7+
expect(a.a).toBe("a");
8+
expect(a.nested.value).toBe(42);
9+
expect(a.nested.a).toBe("a");
10+
expect(a.b).toBe("b");
11+
expect(a.nested.b).toBe("b");
12+
if (step !== 0) {
13+
expect(STATE.random === a.random).toBe(step === 1);
14+
}
15+
STATE.random = a.random;
16+
});

‎test/watchCases/cache/loader-import-module-progress/0/loader.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/** @type {import("../../../../../").PitchLoaderDefinitionFunction} */
2+
exports.pitch = async function (remaining) {
3+
const result = await this.importModule(
4+
`${this.resourcePath}.webpack[javascript/auto]!=!${remaining}`
5+
);
6+
return JSON.stringify(result, null, 2);
7+
};

‎test/watchCases/cache/loader-import-module-progress/0/unrelated.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export const value = 42;

‎test/watchCases/cache/loader-import-module-progress/1/unrelated.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export const value = 24;

0 commit comments

Comments
 (0)