daohualiuxiang
/
vscode
mirror of https://github.com/microsoft/vscode.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip

legomushroom/prompt-file-reference-refactor2
Oleg Solomko 2024-12-22 14:03:10 -08:00
parent 11fced5801
commit 415dd0470c
7 changed files with 236 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/vs/workbench/contrib/chat/common/basePromptParser.ts
View File

@ -5,7 +5,7 @@
// TODO: @legomushroom - cleanup
import { URI } from '../../../../base/common/uri.js';
import { IPromptFileReference, IPromptProvider, TPromptPart } from './basePromptTypes.js';
import { IPromptFileReference, IPromptContentsProvider, TPromptPart } from './basePromptTypes.js';
import { assert, assertNever } from '../../../../base/common/assert.js';
import { Emitter } from '../../../../base/common/event.js';
import { BaseDecoder } from '../../../../base/common/codecs/baseDecoder.js';
@ -79,7 +79,7 @@ export class PromptLine extends Disposable {
if (token instanceof FileReference) {
const fileReference = this.instantiationService
.createInstance(PromptFileReference, token, dirname, seenReferences);
.createInstance(PromptFileReference, token, dirname, [...seenReferences]);
this._tokens.push(fileReference);
@ -164,7 +164,7 @@ const PROMPT_SNIPPETS_CONFIG_KEY: string = 'chat.experimental.prompt-snippets';
/**
* TODO: @legomushroom
*/
export class BasePromptParser<T extends IPromptProvider> extends Disposable {
export class BasePromptParser<T extends IPromptContentsProvider> extends Disposable {
public disposed: boolean = false;
/**

9
src/vs/workbench/contrib/chat/common/basePromptTypes.d.ts vendored
View File

@ -19,14 +19,15 @@ type TOnContentChangedCallback = (streamOrError: ReadableStream<Line> | ParseErr
/**
* TODO: @legomushroom - move to the correct place
*/
export interface IPromptProvider extends IDisposable {
export interface IPromptContentsProvider extends IDisposable {
start(): void;
onContentChanged(callback: TOnContentChangedCallback): IDisposable;
readonly uri: URI;
}
type TPromptPartTypes = 'file-reference';
export interface IPromptPart {
readonly type: 'file-reference';
readonly type: TPromptPartTypes;
readonly range: Range;
readonly text: string;
}
@ -35,11 +36,11 @@ export interface IPromptFileReference extends IPromptPart {
readonly type: 'file-reference';
readonly uri: URI;
readonly path: string;
readonly resolveFailed: boolean | undefined;
readonly errorCondition: ParseError | undefined;
tokensTree: readonly TPromptPart[];
allValidFileReferenceUris: readonly URI[];
validFileReferences: readonly IPromptFileReference[];
readonly resolveFailed: boolean | undefined;
readonly errorCondition: ParseError | undefined;
}
/**

18
src/vs/workbench/contrib/chat/common/filePromptContentProvider.ts
View File

@ -4,9 +4,9 @@
*--------------------------------------------------------------------------------------------*/
import { URI } from '../../../../base/common/uri.js';
import { IPromptProvider } from './basePromptTypes.js';
import { IPromptContentsProvider } from './basePromptTypes.js';
import { BasePromptParser } from './basePromptParser.js';
// import { cancelOnDispose } from './cancelOnDisposeDecorator.js';
import { cancelOnDispose } from './cancelOnDisposeDecorator.js';
import { basename } from '../../../../base/common/resources.js';
import { CancellationError } from '../../../../base/common/errors.js';
import { CancellationToken } from '../../../../base/common/cancellation.js';
@ -21,7 +21,7 @@ import { FileChangesEvent, FileChangeType, IFileService } from '../../../../plat
* TODO: @legomushroom
* TODO: @legomushroom - move to the correct place
*/
export class FilePromptContentProvider extends PromptContentsProviderBase<FileChangesEvent> implements IPromptProvider {
export class FilePromptContentProvider extends PromptContentsProviderBase<FileChangesEvent> implements IPromptContentsProvider {
/**
* TODO: @legomushroom
*/
@ -68,7 +68,7 @@ export class FilePromptContentProvider extends PromptContentsProviderBase<FileCh
/**
* TODO: @legomushroom
*/
// @cancelOnDispose
@cancelOnDispose
protected async getContentsStream(
event: FileChangesEvent | 'full',
cancellationToken?: CancellationToken,
@ -82,17 +82,17 @@ export class FilePromptContentProvider extends PromptContentsProviderBase<FileCh
throw new FileOpenFailed(this.uri, 'Failed to open non-existing file.');
}
// if URI doesn't point to a prompt snippet file, don't try to resolve it
if (!BasePromptParser.isPromptSnippet(this.uri)) {
throw new NotPromptSnippetFile(this.uri);
}
const fileStream = await this.fileService.readFileStream(this.uri);
if (!fileStream) {
throw new FileOpenFailed(this.uri, 'Failed to open file stream.');
}
// if URI doesn't point to a prompt snippet file, don't try to resolve it
if (!BasePromptParser.isPromptSnippet(this.uri)) {
throw new NotPromptSnippetFile(this.uri);
}
if (cancellationToken?.isCancellationRequested) {
fileStream.value.destroy();
throw new CancellationError();

4
src/vs/workbench/contrib/chat/common/promptContentsProviderBase.ts
View File

@ -4,8 +4,8 @@
*--------------------------------------------------------------------------------------------*/
import { URI } from '../../../../base/common/uri.js';
import { IPromptProvider } from './basePromptTypes.js';
import { Emitter } from '../../../../base/common/event.js';
import { IPromptContentsProvider } from './basePromptTypes.js';
import { Disposable } from '../../../../base/common/lifecycle.js';
import { ReadableStream } from '../../../../base/common/stream.js';
import { CancellationError } from '../../../../base/common/errors.js';
@ -38,7 +38,7 @@ import { Line } from '../../../../editor/common/codecs/linesCodec/tokens/line.js
*/
export abstract class PromptContentsProviderBase<
TChangeEvent extends NonNullable<unknown>,
> extends Disposable implements IPromptProvider {
> extends Disposable implements IPromptContentsProvider {
/**
* Internal event emitter for the prompt contents change event. Classes that extend
* this abstract class are responsible to use this emitter to fire the contents change

4
src/vs/workbench/contrib/chat/common/textModelContentsProvider.ts
View File

@ -3,7 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { IPromptProvider } from './basePromptTypes.js';
import { IPromptContentsProvider } from './basePromptTypes.js';
import { ITextModel } from '../../../../editor/common/model.js';
import { PromptContentsProviderBase } from './promptContentsProviderBase.js';
import { Line } from '../../../../editor/common/codecs/linesCodec/tokens/line.js';
@ -14,7 +14,7 @@ import { IModelContentChangedEvent } from '../../../../editor/common/textModelEv
* TODO: @legomushroom
* TODO: @legomushroom - move to a correct place
*/
export class TextModelContentsProvider extends PromptContentsProviderBase<IModelContentChangedEvent> implements IPromptProvider {
export class TextModelContentsProvider extends PromptContentsProviderBase<IModelContentChangedEvent> implements IPromptContentsProvider {
public readonly uri = this.model.uri;
constructor(

2
src/vs/workbench/contrib/chat/common/textModelPromptParser.ts
View File

@ -27,6 +27,6 @@ export class TextModelPromptParser extends BasePromptParser<TextModelContentsPro
* Returns a string representation of this object.
*/
public override toString() {
return `text-editor-prompt:${this.uri.path}`;
return `text-model-prompt:${this.uri.path}`;
}
}

236
src/vs/workbench/contrib/chat/test/common/textModelPromptParser.test.ts
View File

@ -4,14 +4,26 @@
*--------------------------------------------------------------------------------------------*/
import assert from 'assert';
import { URI } from '../../../../../base/common/uri.js';
import { Schemas } from '../../../../../base/common/network.js';
import { extUri } from '../../../../../base/common/resources.js';
import { randomInt } from '../../../../../base/common/numbers.js';
import { Range } from '../../../../../editor/common/core/range.js';
import { isWindows } from '../../../../../base/common/platform.js';
import { DisposableStore } from '../../../../../base/common/lifecycle.js';
import { TErrorCondition } from '../../common/basePromptParser.js';
import { ITextModel } from '../../../../../editor/common/model.js';
import { IPromptFileReference } from '../../common/basePromptTypes.js';
import { NullLogService } from '../../../../../platform/log/common/log.js';
import { IFileService } from '../../../../../platform/files/common/files.js';
import { TextModelPromptParser } from '../../common/textModelPromptParser.js';
import { FileService } from '../../../../../platform/files/common/fileService.js';
import { Disposable, DisposableStore } from '../../../../../base/common/lifecycle.js';
import { FileReference } from '../../common/codecs/chatPromptCodec/tokens/fileReference.js';
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js';
import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
import { createModelServices, instantiateTextModel } from '../../../../../editor/test/common/testTextModel.js';
import { InMemoryFileSystemProvider } from '../../../../../platform/files/common/inMemoryFilesystemProvider.js';
import { FileOpenFailed } from '../../common/promptFileReferenceErrors.js';
/**
* Helper function that allows to await for a specified amount of time.
@ -22,6 +34,181 @@ const wait = (ms: number): Promise<void> => {
return new Promise(resolve => setTimeout(resolve, ms));
};
/**
* Helper function that allows to await for a random amount of time.
* @param maxMs The `maximum` amount of time to wait, in milliseconds.
* @param minMs [`optional`] The `minimum` amount of time to wait, in milliseconds.
* TODO: @legomushroom - move to a common utility
*/
const waitRandom = (maxMs: number, minMs: number = 0): Promise<void> => {
return wait(randomInt(maxMs, minMs));
};
/**
* Represents a file reference with an expected
* error condition value for testing purposes.
* TODO: @legomushroom - move to a common utility
*/
class ExpectedReference {
/**
* URI component of the expected reference.
*/
public readonly uri: URI;
constructor(
dirname: URI,
public readonly lineToken: FileReference,
public readonly errorCondition?: TErrorCondition,
) {
this.uri = extUri.resolvePath(dirname, lineToken.path);
}
/**
* String representation of the expected reference.
*/
public toString(): string {
return `#file:${this.uri.path}`;
}
}
/**
* A reusable test utility to test the `TextModelParser` class.
*/
class TestTextModelParser extends Disposable {
/**
* Reference to the test instance of the the `TextModel`.
*/
private readonly model: ITextModel;
constructor(
modelUri: URI,
private readonly modelContent: string,
private readonly expectedReferences: ExpectedReference[],
@IInstantiationService private readonly initService: IInstantiationService,
@IFileService private readonly fileService: IFileService,
) {
super();
// create in-memory file system
const fileSystemProvider = this._register(new InMemoryFileSystemProvider());
this._register(this.fileService.registerProvider(Schemas.file, fileSystemProvider));
this.model = this._register(instantiateTextModel(
initService,
this.modelContent,
null,
undefined,
modelUri,
));
}
/**
* Directory containing the test model.
*/
public get dirname(): URI {
return extUri.dirname(this.model.uri);
}
/**
* Run the test.
*/
public async run() {
// TODO: @legomushroom - add description
// TODO: @legomushroom - test without the delay
await waitRandom(5);
// start resolving references for the specified root file
const parser = this._register(
this.initService.createInstance(
TextModelPromptParser,
this.model,
[],
),
).start();
// TODO: @legomushroom - add description
await wait(100);
// resolve the root file reference including all nested references
const resolvedReferences: readonly (IPromptFileReference | undefined)[] = parser.tokensTree;
for (let i = 0; i < this.expectedReferences.length; i++) {
const expectedReference = this.expectedReferences[i];
const resolvedReference = resolvedReferences[i];
assert(
(resolvedReference) &&
(resolvedReference.uri.toString() === expectedReference.uri.toString()),
[
`Expected ${i}th resolved reference URI to be '${expectedReference.uri}'`,
`got '${resolvedReference?.uri}'.`,
].join(', '),
);
// assert(
// resolvedReference.token.equals(expectedReference.lineToken),
// [
// `Expected ${i}th resolved reference token to be '${expectedReference.lineToken}'`,
// `got '${resolvedReference.token}'.`,
// ].join(', '),
// );
if (expectedReference.errorCondition === undefined) {
assert(
resolvedReference.errorCondition === undefined,
[
`Expected ${i}th error condition to be 'undefined'`,
`got '${resolvedReference.errorCondition}'.`,
].join(', '),
);
continue;
}
assert(
expectedReference.errorCondition.equal(resolvedReference.errorCondition),
[
`Expected ${i}th error condition to be '${expectedReference.errorCondition}'`,
`got '${resolvedReference.errorCondition}'.`,
].join(', '),
);
}
assert.strictEqual(
resolvedReferences.length,
this.expectedReferences.length,
[
`\nExpected(${this.expectedReferences.length}): [\n ${this.expectedReferences.join('\n ')}\n]`,
`Received(${resolvedReferences.length}): [\n ${resolvedReferences.join('\n ')}\n]`,
].join('\n')
);
}
}
/**
* Create expected file reference for testing purposes.
*
* @param filePath The expected path of the file reference (without the `#file:` prefix).
* @param lineNumber The expected line number of the file reference.
* @param startColumnNumber The expected start column number of the file reference.
* TODO: @legomushroom - move to a common utility
*/
const createTestFileReference = (
filePath: string,
lineNumber: number,
startColumnNumber: number,
): FileReference => {
const range = new Range(
lineNumber,
startColumnNumber,
lineNumber,
startColumnNumber + `#file:${filePath}`.length,
);
return new FileReference(range, filePath);
};
suite('TextModelPromptParser (Unix)', function () {
const testDisposables = ensureNoDisposablesAreLeakedInTestSuite();
@ -30,34 +217,39 @@ suite('TextModelPromptParser (Unix)', function () {
this.skip();
}
const disposables = new DisposableStore();
const disposables = testDisposables.add(new DisposableStore());
const instantiationService = createModelServices(disposables);
const model = instantiateTextModel(instantiationService, 'My First Line\r\nMy \t#file:./some-file.md Line\r\nMy Third Line');
model.registerDisposable(disposables);
const nullLogService = testDisposables.add(new NullLogService());
const nullFileService = testDisposables.add(new FileService(nullLogService));
instantiationService.stub(IFileService, nullFileService);
testDisposables.add(model);
const rootUri = URI.file('/root');
await wait(100);
const test = testDisposables.add(instantiationService.createInstance(
TestTextModelParser,
extUri.joinPath(rootUri, 'file.txt'),
`My First Line\r\nMy \t#file:./some-file.md Line\t\t some content\t#file:/root/some-file.md\r\nMy Third Line`,
[
new ExpectedReference(
rootUri,
createTestFileReference('./some-file.md', 2, 4),
new FileOpenFailed(
URI.joinPath(rootUri, './some-file.md'),
'Failed to open file',
),
),
new ExpectedReference(
rootUri,
createTestFileReference('./some-file.md', 2, 4),
new FileOpenFailed(
URI.joinPath(rootUri, './some-file.md'),
'Failed to open file',
),
),
],
));
const parser = testDisposables.add(
instantiationService.createInstance(TextModelPromptParser, model, []),
);
parser.start();
// TODO: @legomushroom - add description
await wait(100);
const tokens = parser.tokens;
assert(
tokens.length === 1,
`Expected 1 token, but got ${tokens.length}.`,
);
await test.run();
});
});