From 6ad1cc2034d2719fdcafc561a915e54da0b3c179 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Dec 2019 16:51:15 -0700 Subject: [PATCH 01/15] Factor out RawFileSystem. --- src/client/common/platform/fileSystem.ts | 270 +++- src/client/common/platform/types.ts | 5 +- .../platform/filesystem.functional.test.ts | 1376 ++++++++++------- 3 files changed, 1010 insertions(+), 641 deletions(-) diff --git a/src/client/common/platform/fileSystem.ts b/src/client/common/platform/fileSystem.ts index fadeccaaf929..72d369153aba 100644 --- a/src/client/common/platform/fileSystem.ts +++ b/src/client/common/platform/fileSystem.ts @@ -8,6 +8,7 @@ import * as fs from 'fs-extra'; import * as glob from 'glob'; import { inject, injectable } from 'inversify'; import { promisify } from 'util'; +import * as vscode from 'vscode'; import { createDeferred } from '../utils/async'; import { noop } from '../utils/misc'; import { FileSystemPaths, FileSystemPathUtils } from './fs-paths'; @@ -16,9 +17,11 @@ import { TemporaryFileSystem } from './fs-temp'; import { FileStat, FileType, IFileSystem, IFileSystemPaths, IPlatformService, - TemporaryFile + ReadStream, TemporaryFile, WriteStream } from './types'; +const ENCODING: string = 'utf8'; + const globAsync = promisify(glob); // This helper function determines the file type of the given stats @@ -81,11 +84,176 @@ export function convertStat(old: fs.Stats, filetype: FileType): FileStat { }; } +//========================================== +// "raw" filesystem + +// This is the parts of the vscode.workspace.fs API that we use here. +// See: https://code.visualstudio.com/api/references/vscode-api#FileSystem +// Note that we have used all the API functions *except* "rename()". +interface IVSCodeFileSystemAPI { + stat(uri: vscode.Uri): Thenable; +} + +// This is the parts of the 'fs-extra' module that we use in RawFileSystem. +interface IRawFSExtra { + lstat(filename: string): Promise; + readdir(dirname: string): Promise; + readFile(filename: string): Promise; + readFile(filename: string, encoding: string): Promise; + mkdirp(dirname: string): Promise; + chmod(filePath: string, mode: string | number): Promise; + rename(src: string, tgt: string): Promise; + writeFile(filename: string, data: {}, options: {}): Promise; + appendFile(filename: string, data: {}): Promise; + unlink(filename: string): Promise; + rmdir(dirname: string): Promise; + + // non-async + statSync(filename: string): fs.Stats; + readFileSync(path: string, encoding: string): string; + createReadStream(filename: string): ReadStream; + createWriteStream(filename: string): WriteStream; +} + +interface IRawPath { + join(...paths: string[]): string; +} + +// Later we will drop "FileSystem", switching usage to +// "FileSystemUtils" and then rename "RawFileSystem" to "FileSystem". + +// The low-level filesystem operations used by the extension. +export class RawFileSystem { + // prettier-ignore + constructor( + protected readonly paths: IRawPath, + protected readonly vscfs: IVSCodeFileSystemAPI, + protected readonly fsExtra: IRawFSExtra + ) { } + + // Create a new object using common-case default values. + // prettier-ignore + public static withDefaults( + paths?: IRawPath, + vscfs?: IVSCodeFileSystemAPI, + fsExtra?: IRawFSExtra + ): RawFileSystem{ + // prettier-ignore + return new RawFileSystem( + paths || FileSystemPaths.withDefaults(), + vscfs || vscode.workspace.fs, + fsExtra || fs + ); + } + + public readData(filename: string): Promise { + return this.fsExtra.readFile(filename); + } + + // Return the UTF8-decoded text of the file. + public async readText(filename: string): Promise { + return this.fsExtra.readFile(filename, ENCODING); + } + + // Write the UTF8-encoded text to the file. + public async writeText(filename: string, text: string): Promise { + await this.fsExtra.writeFile(filename, text, { encoding: ENCODING }); + } + + public async appendText(filename: string, text: string): Promise { + return this.fsExtra.appendFile(filename, text); + } + + public async rmtree(dirname: string): Promise { + return this.fsExtra.rmdir(dirname); + } + + public async rmfile(filename: string): Promise { + return this.fsExtra.unlink(filename); + } + + public async stat(filename: string): Promise { + // Note that, prior to the November release of VS Code, + // stat.ctime was always 0. + // See: https://github.com/microsoft/vscode/issues/84525 + const uri = vscode.Uri.file(filename); + return this.vscfs.stat(uri); + } + + public async listdir(dirname: string): Promise<[string, FileType][]> { + const files = await this.fsExtra.readdir(dirname); + const promises = files.map(async basename => { + const filename = this.paths.join(dirname, basename); + const fileType = await getFileType(filename); + return [filename, fileType] as [string, FileType]; + }); + return Promise.all(promises); + } + + public async mkdirp(dirname: string): Promise { + return this.fsExtra.mkdirp(dirname); + } + + public async copyFile(src: string, dest: string): Promise { + const deferred = createDeferred(); + // prettier-ignore + const rs = this.fsExtra.createReadStream(src) + .on('error', err => { + deferred.reject(err); + }); + // prettier-ignore + const ws = this.fsExtra.createWriteStream(dest) + .on('error', err => { + deferred.reject(err); + }) + .on('close', () => { + deferred.resolve(); + }); + rs.pipe(ws); + return deferred.promise; + } + + public async move(src: string, tgt: string) { + await this.fsExtra.rename(src, tgt); + } + + public async chmod(filename: string, mode: string | number): Promise { + return this.fsExtra.chmod(filename, mode); + } + + public async lstat(filename: string): Promise { + const stat = await this.fsExtra.lstat(filename); + // Note that, unlike stat(), lstat() does not include the type + // of the symlink's target. + const fileType = convertFileType(stat); + return convertStat(stat, fileType); + } + + //**************************** + // non-async + + public readTextSync(filename: string): string { + return this.fsExtra.readFileSync(filename, ENCODING); + } + + public createReadStream(filename: string): ReadStream { + return this.fsExtra.createReadStream(filename); + } + + public createWriteStream(filename: string): WriteStream { + return this.fsExtra.createWriteStream(filename); + } +} + +//========================================== +// filesystem "utils" (& legacy aliases) + @injectable() export class FileSystem implements IFileSystem { private readonly paths: IFileSystemPaths; private readonly pathUtils: FileSystemPathUtils; private readonly tmp: TemporaryFileSystem; + private readonly raw: RawFileSystem; // prettier-ignore constructor( @inject(IPlatformService) platformService: IPlatformService @@ -96,6 +264,7 @@ export class FileSystem implements IFileSystem { ); this.pathUtils = FileSystemPathUtils.withDefaults(this.paths); this.tmp = TemporaryFileSystem.withDefaults(); + this.raw = RawFileSystem.withDefaults(this.paths); } //================================= @@ -112,107 +281,70 @@ export class FileSystem implements IFileSystem { //================================= // "raw" operations - public async stat(filePath: string): Promise { - // Do not import vscode directly, as this isn't available in the Debugger Context. - // If stat is used in debugger context, it will fail, however theres a separate PR that will resolve this. - // tslint:disable-next-line: no-require-imports - const vscode = require('vscode'); - // Note that, prior to the November release of VS Code, - // stat.ctime was always 0. - // See: https://github.com/microsoft/vscode/issues/84525 - return vscode.workspace.fs.stat(vscode.Uri.file(filePath)); + public async stat(filename: string): Promise { + return this.raw.stat(filename); } + public async lstat(filename: string): Promise { - const stat = await fs.lstat(filename); - // Note that, unlike stat(), lstat() does not include the type - // of the symlink's target. - const fileType = convertFileType(stat); - return convertStat(stat, fileType); + return this.raw.lstat(filename); } - // Return the UTF8-decoded text of the file. - public readFile(filePath: string): Promise { - return fs.readFile(filePath, 'utf8'); + public async readFile(filePath: string): Promise { + return this.raw.readText(filePath); } public readFileSync(filePath: string): string { - return fs.readFileSync(filePath, 'utf8'); + return this.raw.readTextSync(filePath); } - public readData(filePath: string): Promise { - return fs.readFile(filePath); + public async readData(filePath: string): Promise { + return this.raw.readData(filePath); } - public async writeFile(filePath: string, data: {}, options: string | fs.WriteFileOptions = { encoding: 'utf8' }): Promise { - await fs.writeFile(filePath, data, options); + public async writeFile(filePath: string, text: string, _options: string | fs.WriteFileOptions = { encoding: 'utf8' }): Promise { + // tslint:disable-next-line:no-suspicious-comment + // TODO (GH-8542) For now we ignore the options, since all call + // sites already match the defaults. Later we will fix the call + // sites. + return this.raw.writeText(filePath, text); } - public createDirectory(directoryPath: string): Promise { - return fs.mkdirp(directoryPath); + public async createDirectory(directoryPath: string): Promise { + return this.raw.mkdirp(directoryPath); } - public deleteDirectory(directoryPath: string): Promise { - const deferred = createDeferred(); - fs.rmdir(directoryPath, err => (err ? deferred.reject(err) : deferred.resolve())); - return deferred.promise; + public async deleteDirectory(directoryPath: string): Promise { + return this.raw.rmtree(directoryPath); } public async listdir(dirname: string): Promise<[string, FileType][]> { - const files = await fs.readdir(dirname); - const promises = files.map(async basename => { - const filename = this.paths.join(dirname, basename); - const fileType = await getFileType(filename); - return [filename, fileType] as [string, FileType]; - }); - return Promise.all(promises); + return this.raw.listdir(dirname); } - public appendFile(filename: string, data: {}): Promise { - return fs.appendFile(filename, data); + public async appendFile(filename: string, text: string): Promise { + return this.raw.appendText(filename, text); } - public copyFile(src: string, dest: string): Promise { - const deferred = createDeferred(); - const rs = fs.createReadStream(src).on('error', err => { - deferred.reject(err); - }); - const ws = fs - .createWriteStream(dest) - .on('error', err => { - deferred.reject(err); - }) - .on('close', () => { - deferred.resolve(); - }); - rs.pipe(ws); - return deferred.promise; + public async copyFile(src: string, dest: string): Promise { + return this.raw.copyFile(src, dest); } - public deleteFile(filename: string): Promise { - const deferred = createDeferred(); - fs.unlink(filename, err => (err ? deferred.reject(err) : deferred.resolve())); - return deferred.promise; + public async deleteFile(filename: string): Promise { + return this.raw.rmfile(filename); } - public chmod(filePath: string, mode: string | number): Promise { - return new Promise((resolve, reject) => { - fileSystem.chmod(filePath, mode, (err: NodeJS.ErrnoException | null) => { - if (err) { - return reject(err); - } - resolve(); - }); - }); + public async chmod(filePath: string, mode: string | number): Promise { + return this.raw.chmod(filePath, mode); } public async move(src: string, tgt: string) { - await fs.rename(src, tgt); + await this.raw.move(src, tgt); } public createReadStream(filePath: string): fileSystem.ReadStream { - return fileSystem.createReadStream(filePath); + return this.raw.createReadStream(filePath); } public createWriteStream(filePath: string): fileSystem.WriteStream { - return fileSystem.createWriteStream(filePath); + return this.raw.createWriteStream(filePath); } //================================= diff --git a/src/client/common/platform/types.ts b/src/client/common/platform/types.ts index e014aaf9f2fd..1541cac86b71 100644 --- a/src/client/common/platform/types.ts +++ b/src/client/common/platform/types.ts @@ -91,6 +91,7 @@ export interface IFileSystemPathUtils { export import FileType = vscode.FileType; export import FileStat = vscode.FileStat; +export type ReadStream = fs.ReadStream; export type WriteStream = fs.WriteStream; export const IFileSystem = Symbol('IFileSystem'); @@ -106,8 +107,8 @@ export interface IFileSystem { listdir(dirname: string): Promise<[string, FileType][]>; readFile(filePath: string): Promise; readData(filePath: string): Promise; - writeFile(filePath: string, data: {}, options?: string | fsextra.WriteFileOptions): Promise; - appendFile(filename: string, data: {}): Promise; + writeFile(filePath: string, text: string | Buffer, options?: string | fsextra.WriteFileOptions): Promise; + appendFile(filename: string, text: string | Buffer): Promise; copyFile(src: string, dest: string): Promise; deleteFile(filename: string): Promise; chmod(path: string, mode: string | number): Promise; diff --git a/src/test/common/platform/filesystem.functional.test.ts b/src/test/common/platform/filesystem.functional.test.ts index 463d052b34b0..4675a8c0f9d8 100644 --- a/src/test/common/platform/filesystem.functional.test.ts +++ b/src/test/common/platform/filesystem.functional.test.ts @@ -6,7 +6,7 @@ import { expect, use } from 'chai'; import * as fs from 'fs-extra'; import * as path from 'path'; -import { convertStat, FileSystem } from '../../../client/common/platform/fileSystem'; +import { convertStat, FileSystem, RawFileSystem } from '../../../client/common/platform/fileSystem'; import { FileSystemPaths, FileSystemPathUtils } from '../../../client/common/platform/fs-paths'; import { PlatformService } from '../../../client/common/platform/platformService'; import { FileType } from '../../../client/common/platform/types'; @@ -23,6 +23,774 @@ const assertArrays = require('chai-arrays'); use(require('chai-as-promised')); use(assertArrays); +suite('FileSystem - raw', () => { + let fileSystem: RawFileSystem; + let fix: FSFixture; + setup(async () => { + fileSystem = RawFileSystem.withDefaults(); + fix = new FSFixture(); + + await assertDoesNotExist(DOES_NOT_EXIST); + }); + teardown(async () => { + await fix.cleanUp(); + await fix.ensureDeleted(DOES_NOT_EXIST); + }); + + suite('readText', () => { + test('returns contents of a file', async () => { + const expected = ''; + const filename = await fix.createFile('x/y/z/spam.py', expected); + + const content = await fileSystem.readText(filename); + + expect(content).to.be.equal(expected); + }); + + test('always UTF-8', async () => { + const expected = '... 😁 ...'; + const filename = await fix.createFile('x/y/z/spam.py', expected); + + const text = await fileSystem.readText(filename); + + expect(text).to.equal(expected); + }); + + test('returns garbage if encoding is UCS-2', async () => { + const filename = await fix.resolve('spam.py'); + // There are probably cases where this would fail too. + // However, the extension never has to deal with non-UTF8 + // cases, so it doesn't matter too much. + const original = '... 😁 ...'; + await fs.writeFile(filename, original, { encoding: 'ucs2' }); + + const text = await fileSystem.readText(filename); + + expect(text).to.equal('.\u0000.\u0000.\u0000 \u0000=�\u0001� \u0000.\u0000.\u0000.\u0000'); + }); + + test('throws an exception if file does not exist', async () => { + const promise = fileSystem.readText(DOES_NOT_EXIST); + + await expect(promise).to.eventually.be.rejected; + }); + }); + + suite('readData', () => { + test('returns contents of a file', async () => { + const text = ''; + const expected = Buffer.from(text, 'utf8'); + const filename = await fix.createFile('x/y/z/spam.py', text); + + const content = await fileSystem.readData(filename); + + expect(content).to.deep.equal(expected); + }); + + test('throws an exception if file does not exist', async () => { + const promise = fileSystem.readData(DOES_NOT_EXIST); + + await expect(promise).to.eventually.be.rejected; + }); + }); + + suite('writeText', () => { + test('creates the file if missing', async () => { + const filename = await fix.resolve('x/y/z/spam.py'); + await assertDoesNotExist(filename); + const data = 'line1\nline2\n'; + + await fileSystem.writeText(filename, data); + + // prettier-ignore + const actual = await fs.readFile(filename) + .then(buffer => buffer.toString()); + expect(actual).to.equal(data); + }); + + test('always UTF-8', async () => { + const filename = await fix.resolve('x/y/z/spam.py'); + const data = '... 😁 ...'; + + await fileSystem.writeText(filename, data); + + // prettier-ignore + const actual = await fs.readFile(filename) + .then(buffer => buffer.toString()); + expect(actual).to.equal(data); + }); + + test('overwrites existing file', async () => { + const filename = await fix.createFile('x/y/z/spam.py', '...'); + const data = 'line1\nline2\n'; + + await fileSystem.writeText(filename, data); + + // prettier-ignore + const actual = await fs.readFile(filename) + .then(buffer => buffer.toString()); + expect(actual).to.equal(data); + }); + }); + + suite('appendText', () => { + test('existing file', async () => { + const orig = 'spamspamspam\n\n'; + const dataToAppend = `Some Data\n${new Date().toString()}\nAnd another line`; + const filename = await fix.createFile('spam.txt', orig); + const expected = `${orig}${dataToAppend}`; + + await fileSystem.appendText(filename, dataToAppend); + + const actual = await fs.readFile(filename, 'utf8'); + expect(actual).to.be.equal(expected); + }); + + test('existing empty file', async () => { + const filename = await fix.createFile('spam.txt'); + const dataToAppend = `Some Data\n${new Date().toString()}\nAnd another line`; + const expected = dataToAppend; + + await fileSystem.appendText(filename, dataToAppend); + + const actual = await fs.readFile(filename, 'utf8'); + expect(actual).to.be.equal(expected); + }); + + test('creates the file if it does not already exist', async () => { + await fileSystem.appendText(DOES_NOT_EXIST, 'spam'); + + const actual = await fs.readFile(DOES_NOT_EXIST, 'utf8'); + expect(actual).to.be.equal('spam'); + }); + + test('fails if not a file', async () => { + const dirname = await fix.createDirectory('spam'); + + const promise = fileSystem.appendText(dirname, 'spam'); + + await expect(promise).to.eventually.be.rejected; + }); + }); + + suite('rmtree', () => { + test('deletes the directory if empty', async () => { + const dirname = await fix.createDirectory('x'); + await assertExists(dirname); + + await fileSystem.rmtree(dirname); + + await assertDoesNotExist(dirname); + }); + + test('fails if the directory is not empty', async () => { + const dirname = await fix.createDirectory('x'); + const filename = await fix.createFile('x/y/z/spam.py'); + await assertExists(filename); + + const promise = fileSystem.rmtree(dirname); + + await expect(promise).to.eventually.be.rejected; + }); + + test('fails if the directory does not exist', async () => { + const promise = fileSystem.rmtree(DOES_NOT_EXIST); + + await expect(promise).to.eventually.be.rejected; + }); + }); + + suite('rmfile', () => { + test('deletes the file', async () => { + const filename = await fix.createFile('x/y/z/spam.py', '...'); + await assertExists(filename); + + await fileSystem.rmfile(filename); + + await assertDoesNotExist(filename); + }); + + test('fails if the file does not exist', async () => { + const promise = fileSystem.rmfile(DOES_NOT_EXIST); + + await expect(promise).to.eventually.be.rejected; + }); + }); + + suite('listdir', () => { + setup(function() { + if (WINDOWS) { + // tslint:disable-next-line:no-suspicious-comment + // TODO(GH-8995) These tests are failing on Windows, + // so we are // temporarily disabling it. + // tslint:disable-next-line:no-invalid-this + return this.skip(); + } + }); + if (SUPPORTS_SYMLINKS) { + test('mixed', async () => { + // Create the target directory and its contents. + const dirname = await fix.createDirectory('x/y/z'); + const file1 = await fix.createFile('x/y/z/__init__.py', ''); + const script = await fix.createFile('x/y/z/__main__.py', '