diff --git a/e2e/harmony/lanes/lane-export-skip-main-history.e2e.ts b/e2e/harmony/lanes/lane-export-skip-main-history.e2e.ts new file mode 100644 index 000000000000..6d42f401aec9 --- /dev/null +++ b/e2e/harmony/lanes/lane-export-skip-main-history.e2e.ts @@ -0,0 +1,203 @@ +import chai, { expect } from 'chai'; +import { Helper } from '@teambit/legacy.e2e-helper'; +import chaiFs from 'chai-fs'; + +chai.use(chaiFs); + +describe('lane export skips main history objects', function () { + this.timeout(0); + let helper: Helper; + before(() => { + helper = new Helper(); + }); + after(() => { + helper.scopeHelper.destroy(); + }); + + describe('lane in scope-L with components in scope-C, main advances, merge main into lane', () => { + let laneScope: string; + let laneScopePath: string; + let mainSnap1: string; + let mainSnap2: string; + let mainSnap3: string; + let mainSnap4: string; + let laneSnap: string; + let mergeSnap: string; + + before(() => { + // scope-C: components' home scope (default remote) + helper.scopeHelper.setWorkspaceWithRemoteScope(); + // scope-L: the lane's home scope (separate) + const newScope = helper.scopeHelper.getNewBareScope(); + laneScope = newScope.scopeName; + laneScopePath = newScope.scopePath; + helper.scopeHelper.addRemoteScope(laneScopePath); + helper.scopeHelper.addRemoteScope(laneScopePath, helper.scopes.remotePath); + helper.scopeHelper.addRemoteScope(helper.scopes.remotePath, laneScopePath); + + // build several main snaps in scope-C + helper.fixtures.populateComponents(1); + helper.command.tagAllWithoutBuild(); + mainSnap1 = helper.command.getHead('comp1'); + helper.command.tagAllWithoutBuild('--unmodified'); + mainSnap2 = helper.command.getHead('comp1'); + helper.command.tagAllWithoutBuild('--unmodified'); + mainSnap3 = helper.command.getHead('comp1'); + helper.command.export(); + + // create the lane in scope-L (off main, so its base is mainSnap3) + helper.command.createLane('dev', `--scope ${laneScope}`); + helper.command.snapAllComponentsWithoutBuild('--unmodified'); + laneSnap = helper.command.getHeadOfLane('dev', 'comp1'); + helper.command.export(); + const laneWorkspace = helper.scopeHelper.cloneWorkspace(); + + // main advances further (more snaps that the lane never saw) + helper.command.switchLocalLane('main'); + helper.command.tagAllWithoutBuild('--unmodified'); + mainSnap4 = helper.command.getHead('comp1'); + helper.command.export(); + + // back to lane, pick up new main, merge main in (no squash; that's the default for lane->merge) + helper.scopeHelper.getClonedWorkspace(laneWorkspace); + helper.command.import(); + helper.command.mergeLane('main', '--auto-merge-resolve theirs'); + mergeSnap = helper.command.getHeadOfLane('dev', 'comp1'); + helper.command.export(); + }); + + describe('what was pushed to lane scope (scope-L)', () => { + it('should contain the merge snap', () => { + expect(() => helper.command.catObject(mergeSnap, false, laneScopePath)).to.not.throw(); + }); + + it('should contain the lane-only snap', () => { + expect(() => helper.command.catObject(laneSnap, false, laneScopePath)).to.not.throw(); + }); + + it('should NOT contain main snaps that the lane was based on', () => { + expect(() => helper.command.catObject(mainSnap1, false, laneScopePath)).to.throw(); + expect(() => helper.command.catObject(mainSnap2, false, laneScopePath)).to.throw(); + expect(() => helper.command.catObject(mainSnap3, false, laneScopePath)).to.throw(); + }); + + it('should NOT contain the new main snap that got merged in', () => { + expect(() => helper.command.catObject(mainSnap4, false, laneScopePath)).to.throw(); + }); + }); + + describe('what stayed on component home scope (scope-C)', () => { + it('should still contain all main snaps', () => { + expect(() => helper.command.catObject(mainSnap1, false, helper.scopes.remotePath)).to.not.throw(); + expect(() => helper.command.catObject(mainSnap2, false, helper.scopes.remotePath)).to.not.throw(); + expect(() => helper.command.catObject(mainSnap3, false, helper.scopes.remotePath)).to.not.throw(); + expect(() => helper.command.catObject(mainSnap4, false, helper.scopes.remotePath)).to.not.throw(); + }); + }); + + describe('fresh consumer imports the lane', () => { + before(() => { + helper.scopeHelper.reInitWorkspace(); + helper.scopeHelper.addRemoteScope(); + helper.scopeHelper.addRemoteScope(laneScopePath); + helper.command.runCmd(`bit lane import ${laneScope}/dev -x`); + }); + + it('bit status should not throw', () => { + expect(() => helper.command.status()).to.not.throw(); + }); + + it('checkout HEAD should not throw', () => { + expect(() => helper.command.checkoutHead()).to.not.throw(); + }); + + it('bit log should not throw and should show the full history including main snaps', () => { + // bit log walks parents into main history. lane scope does not have those snaps, + // so the read path must fetch them from each component's home scope (scope-C) + // transparently. importMissingHistory is server-side; this fallback is client-side. + let log; + expect(() => { + log = helper.command.logParsed('comp1'); + }).to.not.throw(); + const hashes = log.map((l: any) => l.hash); + expect(hashes).to.include(mergeSnap); + expect(hashes).to.include(laneSnap); + expect(hashes).to.include(mainSnap4); + // pre-lane main history should also be reachable + expect(hashes).to.include(mainSnap3); + expect(hashes).to.include(mainSnap2); + expect(hashes).to.include(mainSnap1); + }); + }); + }); + + describe('sanity: regular (non-merge) lane export still includes lane-only history fully', () => { + let laneScope: string; + let laneScopePath: string; + let snap1: string; + let snap2: string; + + before(() => { + helper.scopeHelper.setWorkspaceWithRemoteScope(); + const newScope = helper.scopeHelper.getNewBareScope(); + laneScope = newScope.scopeName; + laneScopePath = newScope.scopePath; + helper.scopeHelper.addRemoteScope(laneScopePath); + helper.scopeHelper.addRemoteScope(laneScopePath, helper.scopes.remotePath); + helper.scopeHelper.addRemoteScope(helper.scopes.remotePath, laneScopePath); + + // create a lane right away (no main tags), so all snaps are lane-origin + helper.command.createLane('dev', `--scope ${laneScope}`); + helper.fixtures.populateComponents(1); + helper.command.snapAllComponentsWithoutBuild(); + snap1 = helper.command.getHeadOfLane('dev', 'comp1'); + helper.command.snapAllComponentsWithoutBuild('--unmodified'); + snap2 = helper.command.getHeadOfLane('dev', 'comp1'); + helper.command.export(); + }); + + it('lane scope should contain all lane-origin snaps', () => { + expect(() => helper.command.catObject(snap1, false, laneScopePath)).to.not.throw(); + expect(() => helper.command.catObject(snap2, false, laneScopePath)).to.not.throw(); + }); + }); + + describe('sanity: existing main-merge with no scope split still works (lane and components share scope)', () => { + let mainSnap: string; + let mergeSnap: string; + + before(() => { + helper.scopeHelper.setWorkspaceWithRemoteScope(); + helper.fixtures.populateComponents(1); + helper.command.tagAllWithoutBuild(); + mainSnap = helper.command.getHead('comp1'); + helper.command.export(); + + helper.command.createLane('dev'); + helper.command.snapAllComponentsWithoutBuild('--unmodified'); + helper.command.export(); + const laneWorkspace = helper.scopeHelper.cloneWorkspace(); + + helper.command.switchLocalLane('main'); + helper.command.tagAllWithoutBuild('--unmodified'); + helper.command.export(); + + helper.scopeHelper.getClonedWorkspace(laneWorkspace); + helper.command.import(); + helper.command.mergeLane('main', '--auto-merge-resolve theirs'); + mergeSnap = helper.command.getHeadOfLane('dev', 'comp1'); + helper.command.export(); + }); + + it('the shared remote scope should contain both the merge snap and the old main snap (same-scope case)', () => { + // When lane scope == component scope, filtering by origin doesn't apply: everything already + // belongs to this scope. Both objects should be present. + expect(() => helper.command.catObject(mergeSnap, false, helper.scopes.remotePath)).to.not.throw(); + expect(() => helper.command.catObject(mainSnap, false, helper.scopes.remotePath)).to.not.throw(); + }); + + it('bit log should work without errors', () => { + expect(() => helper.command.logParsed('comp1')).to.not.throw(); + }); + }); +}); diff --git a/scopes/component/snap-distance/traverse-versions.ts b/scopes/component/snap-distance/traverse-versions.ts index 5f6f5465f02e..18363cd884aa 100644 --- a/scopes/component/snap-distance/traverse-versions.ts +++ b/scopes/component/snap-distance/traverse-versions.ts @@ -245,12 +245,11 @@ export async function getAllVersionParents({ const missingParents = allParentsFromObj.filter((parent) => !versionParents.some((v) => v.hash.isEqual(parent))); if (missingParents.length) { if (missingRefsFromVersionHistory) { - // stops the recursion - throw new Error(`unable to get the full history of "${modelComponent.id()}". -the client sent the following snaps: ${versionParentsFromObjects.map((v) => v.hash.toString()).join(', ')}. -however some of the parents of these snaps are missing from the local scope. -missing snaps: ${missingParents.map((m) => m.toString()).join(', ')} -`); + // Lean-lane-scope mode: missing parents are expected — they live on each component's + // home scope (e.g. main snaps for a lane that's far behind main). The lane scope no + // longer pulls them in. Return what we have; consumers will fetch from origin scopes + // when they need older history. + return versionParents; } // the VersionObject is not up to date. // recursively run this function and try to add these missing parents as heads so then it tries diff --git a/scopes/scope/export/export.main.runtime.ts b/scopes/scope/export/export.main.runtime.ts index e4011b21a3b8..7ec34b185345 100644 --- a/scopes/scope/export/export.main.runtime.ts +++ b/scopes/scope/export/export.main.runtime.ts @@ -33,7 +33,7 @@ import type { DependencyResolverMain } from '@teambit/dependency-resolver'; import { DependencyResolverAspect } from '@teambit/dependency-resolver'; import { persistRemotes, validateRemotes, removePendingDirs } from './export-scope-components'; import { writeLastExport } from './last-export'; -import type { Lane, ModelComponent, ObjectItem, LaneReadmeComponent, BitObject, Ref } from '@teambit/objects'; +import type { Lane, ModelComponent, ObjectItem, LaneReadmeComponent, BitObject, Ref, Version } from '@teambit/objects'; import { ObjectList } from '@teambit/objects'; import { Scope, PersistFailed } from '@teambit/legacy.scope'; import { getAllVersionHashes } from '@teambit/component.snap-distance'; @@ -512,7 +512,47 @@ if the scope name is wrong and you've already snapped/tagged, run "bit reset" to objectList.addIfNotExist(allObjectsData); }; - const refsToExportPerComponent = await getRefsToExportPerComp(); + const filterOutForeignMainOriginRefs = async ( + refsPerComp: { modelComponent: ModelComponent; refs: Ref[] }[] + ): Promise<{ modelComponent: ModelComponent; refs: Ref[] }[]> => { + // when exporting a lane, drop refs that represent main-origin snaps belonging to + // components whose home scope is different from the lane's scope. those snaps already + // live on the component's home scope; pushing them to the lane scope is duplication + // (and the main driver of OOM when a lane is far behind main). + if (!lane) return refsPerComp; + const filtered = await mapSeries(refsPerComp, async ({ modelComponent, refs }) => { + const droppedRefs: string[] = []; + const keptRefs: Ref[] = []; + for (const ref of refs) { + const obj = await ref.load(scope.objects); + if (!obj || obj.getType() !== 'Version') { + keptRefs.push(ref); + continue; + } + const version = obj as Version; + const isLaneOrigin = Boolean(version.origin?.lane); + const isLocallyMutated = Boolean(version.squashed) || Boolean(version.unrelated); + const originScope = version.origin?.id?.scope; + const isForeignComponentScope = Boolean(originScope) && originScope !== remoteNameStr; + if (!isLaneOrigin && isForeignComponentScope && !isLocallyMutated) { + droppedRefs.push(ref.toString()); + continue; + } + keptRefs.push(ref); + } + if (droppedRefs.length) { + this.logger.debug( + `export-scope-components, skipping ${droppedRefs.length} main-origin refs for ${modelComponent + .id() + .toString()} (already on component home scope, not the lane scope)` + ); + } + return keptRefs.length ? { modelComponent, refs: keptRefs } : null; + }); + return compact(filtered); + }; + + const refsToExportPerComponent = await filterOutForeignMainOriginRefs(await getRefsToExportPerComp()); // don't use Promise.all, otherwise, it'll throw "JavaScript heap out of memory" on a large set of data await mapSeries(refsToExportPerComponent, processModelComponent); if (lane) { diff --git a/scopes/scope/objects/models/model-component.ts b/scopes/scope/objects/models/model-component.ts index 47df32eb6224..da20497544d4 100644 --- a/scopes/scope/objects/models/model-component.ts +++ b/scopes/scope/objects/models/model-component.ts @@ -562,6 +562,26 @@ export default class Component extends BitObject { } ); versionsInfo = await getAllVersionsInfo({ modelComponent: this, repo, throws: false, startFrom }); + // Lean-lane-scope fallback: if the lane is on a different scope from the component's + // home scope (e.g. a feature lane for a component that lives on main in another scope), + // older main snaps may not be on the lane scope. Re-fetch without lane so the fetcher + // routes to the component's home scope and brings in the missing main history. + if ( + lane && + lane.scope !== this.scope && + versionsInfo.some((v) => v.error && errorIsTypeOfMissingObject(v.error)) + ) { + await scope.scopeImporter.importWithoutDeps( + ComponentIdList.fromArray([this.toComponentId()]).toVersionLatest(), + { + cache: false, + includeVersionHistory: true, + collectParents: true, + reason: 'to collect logs from component home scope (lean-lane-scope fallback)', + } + ); + versionsInfo = await getAllVersionsInfo({ modelComponent: this, repo, throws: false, startFrom }); + } } catch (err) { logger.error(`collectLogs failed to import ${this.id()} history`, err); } diff --git a/scopes/scope/remote-actions/export-validate.ts b/scopes/scope/remote-actions/export-validate.ts index f04031a02931..bdca71572ee8 100644 --- a/scopes/scope/remote-actions/export-validate.ts +++ b/scopes/scope/remote-actions/export-validate.ts @@ -7,7 +7,6 @@ import { mergeObjects } from '@teambit/export'; import type { Action } from './action'; import { logger } from '@teambit/legacy.logger'; import type { BitObjectList } from '@teambit/objects'; -import { getAllVersionHashes } from '@teambit/component.snap-distance'; type Options = { clientId: string; isResumingExport: boolean }; const NUM_OF_RETRIES = 60; @@ -51,12 +50,15 @@ export class ExportValidate implements Action { const modelComponents = bitObjectList.getComponents(); const externalComponents = modelComponents.filter((comp) => comp.scope !== this.scope.name); if (!externalComponents.length) return; - await this.scope.scopeImporter.importMissingVersionHistory(externalComponents); - // this will throw in case the history is missing - await Promise.all( - externalComponents.map((modelComponent) => - getAllVersionHashes({ modelComponent, repo: this.scope.objects, throws: true }) - ) + // Lean-lane-scope: do NOT import full version-history into the lane scope. The history + // lives on each component's home scope (e.g. main snaps in component-scope, lane-b snaps in + // scope-b). Pulling them into the lane scope was the main source of fat-lane-scope OOM when + // a lane is far behind main. Consumers walking history will resolve missing parents by + // fetching from origin scopes on demand. We also skip the strict getAllVersionHashes check + // for the same reason — incomplete history on the lane scope is now expected. + logger.debug( + `export-validate, skipping importMissingVersionHistory for ${externalComponents.length} external components ` + + `(lean lane scope mode — their history stays on origin scopes)` ); }