From 35e1ee2ecc06cf5cc8eadb2c4c2b53d873e3bad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Moreno?= Date: Tue, 2 Feb 2021 13:42:55 +0100 Subject: [PATCH] ext: extract to disk --- build/ext.js | 58 ++++++++++++++++++++++++++++++----------- build/ext.ts | 65 +++++++++++++++++++++++++++++++++------------- build/package.json | 1 + build/yarn.lock | 9 ++++++- 4 files changed, 99 insertions(+), 34 deletions(-) diff --git a/build/ext.js b/build/ext.js index f35fa6e4938..077ca3c22e1 100644 --- a/build/ext.js +++ b/build/ext.js @@ -26,6 +26,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.getExtensions = void 0; const fs_1 = require("fs"); const path = require("path"); +const util_1 = require("util"); const cp = require("child_process"); const commander_1 = require("commander"); const storage_blob_1 = require("@azure/storage-blob"); @@ -34,8 +35,12 @@ const plimit = require("p-limit"); const colors = require("colors"); const byline = require("byline"); const stream_1 = require("stream"); +const rimraf = require("rimraf"); +const zip = require('gulp-vinyl-zip'); +const vfs = require("vinyl-fs"); const rootPath = path.resolve(path.join(__dirname, '..')); const vsixsPath = path.join(rootPath, '.build', 'vsix'); +const extensionsPath = path.join(rootPath, '.build', 'extensions'); var ExtensionType; (function (ExtensionType) { ExtensionType["Grammar"] = "grammar"; @@ -95,7 +100,8 @@ async function getExtension(extensionPath) { return { name, version, - path: extensionPath, + sourcePath: extensionPath, + installPath: path.join(extensionsPath, name), type, vsixPath: path.join(vsixsPath, vsixName) }; @@ -131,7 +137,7 @@ async function each([cmd, ...args], opts) { continue; } console.log(`👉 ${extension.name}`); - await spawn(cmd, args, { cwd: extension.path }); + await spawn(cmd, args, { cwd: extension.sourcePath }); } } catch (e_1_1) { e_1 = { error: e_1_1 }; } @@ -142,9 +148,30 @@ async function each([cmd, ...args], opts) { finally { if (e_1) throw e_1.error; } } } +async function extractExtension(extension) { + await util_1.promisify(rimraf)(extension.installPath); + await new Promise((c, e) => { + zip.src(extension.vsixPath) + .pipe(new stream_1.Transform({ + objectMode: true, + transform(file, _, cb) { + if (/^extension\//.test(file.relative)) { + file.base += '/extension'; + cb(null, file); + } + else { + cb(); + } + } + })) + .pipe(vfs.dest(extension.installPath)) + .on('error', e) + .on('end', () => c()); + }); +} async function runExtensionCI(extension, service) { const vsixName = `${extension.name}-${extension.version}.vsix`; - const commit = await exec(`git log -1 --format="%H" -- ${extension.path}`, { trim: true }); + const commit = await exec(`git log -1 --format="%H" -- ${extension.sourcePath}`, { trim: true }); const container = service.getContainerClient('extensions'); const blobName = `${commit}/${vsixName}`; const blob = container.getBlobClient(blobName); @@ -152,24 +179,25 @@ async function runExtensionCI(extension, service) { try { await blob.downloadToFile(extension.vsixPath); console.log(`${prefix} Downloaded from cache ${colors.grey(`(${blobName})`)}`); - return; } catch (err) { if (err.statusCode !== 404) { throw err; } + console.log(`${prefix} Cache miss ${colors.grey(`(${blobName})`)}`); + console.log(`${prefix} Building...`); + await spawn(`yarn install --no-progress`, { prefix, shell: true, cwd: extension.sourcePath }); + await spawn(`vsce package --yarn -o ${vsixsPath}`, { prefix, shell: true, cwd: extension.sourcePath }); + if (service.credential instanceof storage_blob_1.AnonymousCredential) { + console.log(`${prefix} Skiping publish VSIX to cache (anonymous access only)`); + } + else { + const blockBlob = await blob.getBlockBlobClient(); + await blockBlob.uploadFile(extension.vsixPath); + console.log(`${prefix} Successfully uploaded VSIX to cache`); + } } - console.log(`${prefix} Cache miss ${colors.grey(`(${blobName})`)}`); - console.log(`${prefix} Building...`); - await spawn(`yarn install --no-progress`, { prefix, shell: true, cwd: extension.path }); - await spawn(`vsce package --yarn -o ${vsixsPath}`, { prefix, shell: true, cwd: extension.path }); - if (service.credential instanceof storage_blob_1.AnonymousCredential) { - console.log(`${prefix} Skiping publish VSIX to cache (anonymous access only)`); - return; - } - const blockBlob = await blob.getBlockBlobClient(); - await blockBlob.uploadFile(extension.vsixPath); - console.log(`${prefix} Successfully uploaded VSIX to cache`); + await extractExtension(extension); } async function ci() { var e_2, _a; diff --git a/build/ext.ts b/build/ext.ts index e7b5b60dfaf..19d2d54c026 100644 --- a/build/ext.ts +++ b/build/ext.ts @@ -5,6 +5,7 @@ import { promises as fs } from 'fs'; import * as path from 'path'; +import { promisify } from 'util'; import * as cp from 'child_process'; import { program } from 'commander'; import { AnonymousCredential, BlobServiceClient, StorageSharedKeyCredential } from '@azure/storage-blob'; @@ -13,9 +14,14 @@ import * as plimit from 'p-limit'; import * as colors from 'colors'; import * as byline from 'byline'; import { Transform, TransformCallback } from 'stream'; +import * as rimraf from 'rimraf'; +const zip = require('gulp-vinyl-zip'); +import * as vfs from 'vinyl-fs'; +import * as File from 'vinyl'; const rootPath = path.resolve(path.join(__dirname, '..')); const vsixsPath = path.join(rootPath, '.build', 'vsix'); +const extensionsPath = path.join(rootPath, '.build', 'extensions'); const enum ExtensionType { Grammar = 'grammar', @@ -26,7 +32,8 @@ const enum ExtensionType { interface IExtension { readonly name: string; readonly version: string; - readonly path: string; + readonly sourcePath: string; + readonly installPath: string; readonly type: ExtensionType; readonly vsixPath: string; } @@ -106,7 +113,8 @@ async function getExtension(extensionPath: string): Promise { return { name, version, - path: extensionPath, + sourcePath: extensionPath, + installPath: path.join(extensionsPath, name), type, vsixPath: path.join(vsixsPath, vsixName) }; @@ -142,13 +150,34 @@ async function each([cmd, ...args]: string[], opts: { type?: string }) { } console.log(`👉 ${extension.name}`); - await spawn(cmd, args, { cwd: extension.path }); + await spawn(cmd, args, { cwd: extension.sourcePath }); } } +async function extractExtension(extension: IExtension): Promise { + await promisify(rimraf)(extension.installPath); + await new Promise((c, e) => { + zip.src(extension.vsixPath) + .pipe(new Transform({ + objectMode: true, + transform(file: File, _, cb) { + if (/^extension\//.test(file.relative)) { + file.base += '/extension'; + cb(null, file); + } else { + cb(); + } + } + })) + .pipe(vfs.dest(extension.installPath)) + .on('error', e) + .on('end', () => c()); + }); +} + async function runExtensionCI(extension: IExtension, service: BlobServiceClient): Promise { const vsixName = `${extension.name}-${extension.version}.vsix`; - const commit = await exec(`git log -1 --format="%H" -- ${extension.path}`, { trim: true }); + const commit = await exec(`git log -1 --format="%H" -- ${extension.sourcePath}`, { trim: true }); const container = service.getContainerClient('extensions'); const blobName = `${commit}/${vsixName}`; const blob = container.getBlobClient(blobName); @@ -157,26 +186,26 @@ async function runExtensionCI(extension: IExtension, service: BlobServiceClient) try { await blob.downloadToFile(extension.vsixPath); console.log(`${prefix} Downloaded from cache ${colors.grey(`(${blobName})`)}`); - return; } catch (err) { if (err.statusCode !== 404) { throw err; } + + console.log(`${prefix} Cache miss ${colors.grey(`(${blobName})`)}`); + console.log(`${prefix} Building...`); + await spawn(`yarn install --no-progress`, { prefix, shell: true, cwd: extension.sourcePath }); + await spawn(`vsce package --yarn -o ${vsixsPath}`, { prefix, shell: true, cwd: extension.sourcePath }); + + if (service.credential instanceof AnonymousCredential) { + console.log(`${prefix} Skiping publish VSIX to cache (anonymous access only)`); + } else { + const blockBlob = await blob.getBlockBlobClient(); + await blockBlob.uploadFile(extension.vsixPath); + console.log(`${prefix} Successfully uploaded VSIX to cache`); + } } - console.log(`${prefix} Cache miss ${colors.grey(`(${blobName})`)}`); - console.log(`${prefix} Building...`); - await spawn(`yarn install --no-progress`, { prefix, shell: true, cwd: extension.path }); - await spawn(`vsce package --yarn -o ${vsixsPath}`, { prefix, shell: true, cwd: extension.path }); - - if (service.credential instanceof AnonymousCredential) { - console.log(`${prefix} Skiping publish VSIX to cache (anonymous access only)`); - return; - } - - const blockBlob = await blob.getBlockBlobClient(); - await blockBlob.uploadFile(extension.vsixPath); - console.log(`${prefix} Successfully uploaded VSIX to cache`); + await extractExtension(extension); } async function ci(): Promise { diff --git a/build/package.json b/build/package.json index 7de0c8f734c..509540c1212 100644 --- a/build/package.json +++ b/build/package.json @@ -50,6 +50,7 @@ "mkdirp": "^1.0.4", "p-limit": "^3.1.0", "plist": "^3.0.1", + "rimraf": "^3.0.2", "source-map": "0.6.1", "typescript": "4.2.0-dev.20201207", "vsce": "1.48.0", diff --git a/build/yarn.lock b/build/yarn.lock index a111c831be1..0d7f801a4b6 100644 --- a/build/yarn.lock +++ b/build/yarn.lock @@ -1038,7 +1038,7 @@ glob@^7.0.6: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.1.6: +glob@^7.1.3, glob@^7.1.6: version "7.1.6" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== @@ -1528,6 +1528,13 @@ request@^2.86.0: tunnel-agent "^0.6.0" uuid "^3.3.2" +rimraf@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"