Initial commit

Co-Authored-By: Eric Tuvesson <eric.tuvesson@gmail.com>
Co-Authored-By: mikaeltellhed <2311083+mikaeltellhed@users.noreply.github.com>
Co-Authored-By: kotte <14197736+mrtamagotchi@users.noreply.github.com>
Co-Authored-By: Anders Larsson <64838990+anders-topp@users.noreply.github.com>
Co-Authored-By: Johan  <4934465+joolsus@users.noreply.github.com>
Co-Authored-By: Tore Knudsen <18231882+torekndsn@users.noreply.github.com>
Co-Authored-By: victoratndl <99176179+victoratndl@users.noreply.github.com>

scripts/noodl-editor/build-editor.ts

@@ -0,0 +1,103 @@
+import { execSync } from 'child_process';
+import path from 'path';
+
+// import { gitSimpleDownload } from "./git/download-git";
+
+// HACK :)
+import { FileSystemNode } from '../../packages/noodl-platform-node/src/filesystem-node';
+import { getCurrentPlatform } from '../helper';
+
+(async function () {
+  const filesystem = new FileSystemNode();
+
+  // Inputs
+  const WORKSPACE_PATH = path.resolve(__dirname, '../..');
+  const TARGET_PLATFORM = process.env.TARGET_PLATFORM || getCurrentPlatform();
+
+  // Variables
+  const noodlEditorPath = path.join(WORKSPACE_PATH, 'packages', 'noodl-editor');
+  const pkgPath = path.join(noodlEditorPath, 'package.json');
+  const configPath = path.join(noodlEditorPath, 'src/shared/config/config.js');
+  const configDistPath = path.join(noodlEditorPath, 'src/shared/config/config-dist.js');
+
+  const [platform, arch] = TARGET_PLATFORM.trim().split('-');
+  // @ts-expect-error TODO: Add validation on the input.
+  const target: BuildTarget = { platform, arch };
+
+  // Debug Configuration
+  console.log('--- Configuration');
+  console.log('> WORKSPACE_PATH: ', WORKSPACE_PATH);
+  console.log('> TARGET_PLATFORM: ', TARGET_PLATFORM);
+  console.log('---');
+  console.log('> noodlEditorPath: ', noodlEditorPath);
+  console.log('> pkgPath: ', pkgPath);
+  console.log('> configPath: ', configPath);
+  console.log('> configDistPath: ', configDistPath);
+  console.log('---');
+
+  // Update "package.json"
+  console.log("--- Update 'package.json' ...");
+  {
+    // const rawGitBranch = execSync('git rev-parse --abbrev-ref HEAD', {
+    //   env: process.env
+    // });
+    // const gitBranch = rawGitBranch.toString().trim().replace(/\//, '-');
+
+    const pkg = await filesystem.readJson(pkgPath);
+    pkg.main = 'src/main/main.bundle.js';
+    // TODO: Set our own new versionTag
+    pkg.versionTag = undefined;
+    // pkg.build.publish.url = getPublishUrl() + `/${TARGET_PLATFORM}`;
+    // console.log('> Auto update URL:', pkg.build.publish.url);
+
+    await filesystem.writeJson(pkgPath, pkg);
+  }
+
+  // Replace `config.js`
+  await filesystem.removeFile(configPath);
+  await filesystem.copyFile(configDistPath, configPath);
+
+  // Yup, arm64 vs x64 battle going on here ...
+  console.log("--- Delete 'node_modules' ...");
+  execSync('npx rimraf ./node_modules', {
+    stdio: 'inherit',
+    env: {
+      ...process.env
+    }
+  });
+
+  // Install dependencies
+  // NOTE: Getting error "Cannot set properties of null (setting 'dev')" here,
+  //       It basically means that some package is not relative to this path.
+  console.log("--- Run 'npm install' ...");
+  execSync(`npm install --arch=${arch} --scope Noodl`, {
+    stdio: 'inherit',
+    env: process.env
+  });
+  console.log("--- 'npm install' done!");
+
+  // NOTE: npm install --arch=  does this too
+  // // Download git natives for the targeted platform
+  // const gitDir = path.resolve("../../node_modules/dugite/git");
+  // await gitSimpleDownload({
+  //   outputPath: gitDir,
+  //   architecture: target.arch,
+  //   platform: target.platform,
+  // });
+
+  // Build
+  // Build: Replace "dugite"
+  // Build: Replace "desktop-trampoline"
+  console.log("--- Run 'npm run build' ...");
+  execSync('npx lerna exec --scope Noodl -- npm run build', {
+    stdio: 'inherit',
+    env: {
+      ...process.env,
+      TARGET_PLATFORM
+    }
+  });
+  console.log("--- 'npm run build' done!");
+
+  // TODO: Create a JSON with metadata example data updated in package.json
+  // TODO: Create a dump of all npm packages + sizes
+})();

scripts/noodl-editor/build-viewer.ts

@@ -0,0 +1,77 @@
+import { execSync } from 'child_process';
+import path from 'path';
+
+// HACK :)
+import { FileSystemNode } from '../../packages/noodl-platform-node/src/filesystem-node';
+
+async function buildViewer({ name, relativePath }) {
+  // Inputs
+  const WORKSPACE_PATH = path.resolve(__dirname, '../..');
+
+  // Variables
+  const noodlViewerPath = path.join(WORKSPACE_PATH, 'packages/' + relativePath);
+  const noodlEditorPath = path.join(WORKSPACE_PATH, 'packages', 'noodl-editor');
+  const destination = path.join(noodlEditorPath, 'src', 'external');
+
+  // Debug Configuration
+  console.log('--- Configuration');
+  console.log('> WORKSPACE_PATH: ', WORKSPACE_PATH);
+  console.log('---');
+  console.log('> noodlViewerPath: ', noodlViewerPath);
+  console.log('> noodlEditorPath: ', noodlEditorPath);
+  console.log('> destination: ', destination);
+  console.log('---');
+
+  // Install dependencies
+  console.log("--- Run 'npm install' (cwd: ", noodlViewerPath, ') ...');
+  execSync('npm i', {
+    cwd: noodlViewerPath,
+    stdio: 'inherit',
+    env: {
+      ...process.env
+    }
+  });
+  console.log("--- 'npm install' done!");
+
+  // Build
+  console.log("--- Run 'npm run build' (cwd: ", noodlViewerPath, ') ...');
+  execSync(`npx lerna exec --scope ${name} -- npm run build`, {
+    cwd: WORKSPACE_PATH,
+    stdio: 'inherit',
+    env: {
+      ...process.env,
+      OUTPUT_PATH: destination
+    }
+  });
+  console.log("--- 'npm run build' done!");
+
+  // Display result
+  const filesystem = new FileSystemNode();
+  const files = await filesystem.listDirectoryFiles(destination);
+  console.log('--- List build files:');
+  files.forEach((file, index) => {
+    console.log(index, file.fullPath);
+  });
+
+  // Clean up
+  console.log('--- Clean up ...');
+  execSync(`npx lerna clean --yes --scope ${name}`, {
+    stdio: 'inherit',
+    env: {
+      ...process.env
+    }
+  });
+  console.log('--- Clean up done!');
+}
+
+(async function () {
+  await buildViewer({
+    relativePath: 'noodl-viewer-react',
+    name: '@noodl/noodl-viewer-react'
+  });
+
+  await buildViewer({
+    relativePath: 'noodl-viewer-cloud',
+    name: '@noodl/cloud-runtime'
+  });
+})();

scripts/noodl-editor/git/clean-up.ts

@@ -0,0 +1,32 @@
+import { unlinkSync } from 'fs-extra';
+import path from 'path';
+
+export function gitCleanUp(options: { gitDir: string; architecture: string }) {
+  if (process.platform === 'win32') {
+    console.log('Noodl Build: Cleaning unneeded Git components…');
+    const files = [
+      'Bitbucket.Authentication.dll',
+      'GitHub.Authentication.exe',
+      'Microsoft.Alm.Authentication.dll',
+      'Microsoft.Alm.Git.dll',
+      'Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll',
+      'Microsoft.IdentityModel.Clients.ActiveDirectory.dll',
+      'Microsoft.Vsts.Authentication.dll',
+      'git-askpass.exe',
+      'git-credential-manager.exe',
+      'WebView2Loader.dll'
+    ];
+
+    const mingwFolder = options.architecture === 'x64' ? 'mingw64' : 'mingw32';
+    const gitCoreDir = path.join(options.gitDir, mingwFolder, 'libexec', 'git-core');
+
+    for (const file of files) {
+      const filePath = path.join(gitCoreDir, file);
+      try {
+        unlinkSync(filePath);
+      } catch (err) {
+        // probably already cleaned up
+      }
+    }
+  }
+}

scripts/noodl-editor/git/config-git.ts

@@ -0,0 +1,67 @@
+import URL from 'url';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+
+export function getGitConfig(options: { platform: string; architecture: string }) {
+  const embeddedGit = require('dugite/script/embedded-git.json');
+
+  const config = {
+    source: '',
+    checksum: '',
+    fileName: '',
+    tempFile: ''
+  };
+
+  let arch = options.architecture;
+
+  if (process.env.npm_config_arch) {
+    // If a specific npm_config_arch is set, we use that one instead of the OS arch (to support cross compilation)
+    console.log('npm_config_arch detected: ' + process.env.npm_config_arch);
+    arch = process.env.npm_config_arch;
+  }
+
+  if (options.platform === 'win32' && arch === 'arm64') {
+    // Use the Dugite Native ia32 package for Windows arm64 (arm64 can run 32-bit code through emulation)
+    console.log('Downloading 32-bit Dugite Native for Windows arm64');
+    arch = 'ia32';
+  }
+
+  // Os.arch() calls it x32, we use x86 in actions, dugite-native calls it x86 and our embedded-git.json calls it ia32
+  if (arch === 'x32' || arch === 'x86') {
+    arch = 'ia32';
+  }
+
+  const key = `${options.platform}-${arch}`;
+
+  const entry = embeddedGit[key];
+
+  if (entry != null) {
+    config.checksum = entry.checksum;
+    config.source = entry.url;
+  } else {
+    console.log(`No embedded Git found for ${options.platform} and architecture ${arch}`);
+  }
+
+  if (config.source !== '') {
+    // compute the filename from the download source
+    const url = URL.parse(config.source);
+    const pathName = url.pathname;
+    const index = pathName.lastIndexOf('/');
+    config.fileName = pathName.substring(index + 1);
+
+    const cacheDirEnv = process.env.DUGITE_CACHE_DIR;
+
+    const cacheDir = cacheDirEnv ? path.resolve(cacheDirEnv) : os.tmpdir();
+
+    try {
+      fs.statSync(cacheDir);
+    } catch (e) {
+      fs.mkdirSync(cacheDir);
+    }
+
+    config.tempFile = path.join(cacheDir, config.fileName);
+  }
+
+  return config;
+}

scripts/noodl-editor/git/download-git.ts

@@ -0,0 +1,219 @@
+import fs from 'fs';
+import got from 'got';
+import ProgressBar from 'progress';
+import mkdirp from 'mkdirp';
+import checksum from 'checksum';
+import rimraf from 'rimraf';
+import tar from 'tar';
+import zlib from 'zlib';
+import { getGitConfig } from './config-git';
+import { gitCleanUp } from './clean-up';
+
+export interface GitDownloadOptions {
+  outputPath: string;
+  tempFile: string;
+  source: string;
+  checksum: string;
+  platform: string;
+  architecture: string;
+}
+
+export async function gitSimpleDownload(options: { outputPath: string; platform: string; architecture: string }) {
+  console.log('Noodl Build: -- Begin Git');
+  console.log('> outputPath: ', options.outputPath);
+  console.log('> architecture: ', options.architecture);
+  console.log('> platform: ', options.platform);
+  console.log('--');
+
+  console.log(`Noodl Build: Create git config...`);
+  const config = getGitConfig({
+    architecture: options.architecture,
+    platform: options.platform
+  });
+
+  console.log('> config.checksum: ', config.checksum);
+  console.log('> config.fileName: ', config.fileName);
+  console.log('> config.source: ', config.source);
+  console.log('> config.tempFile: ', config.tempFile);
+  console.log('> options.outputPath: ', options.outputPath);
+
+  const targetName = `${options.platform}-${options.architecture}`;
+  console.log(`Noodl Build: Download git natives for ${targetName}...`);
+
+  // Download git natives for the targeted platform
+  await gitDownload({
+    ...config,
+    ...options
+  });
+
+  console.log(`Noodl Build: Clean up git natives...`);
+
+  // Clean up the git files that we don't need
+  gitCleanUp({
+    gitDir: options.outputPath,
+    architecture: options.architecture
+  });
+
+  console.log('Noodl Build: -- End Git');
+}
+
+function verifyFile(file: string, matchChecksum: string) {
+  return new Promise<boolean>((resolve) => {
+    checksum.file(file, { algorithm: 'sha256' }, async (_, hash) => {
+      const match = hash === matchChecksum;
+
+      if (!match) {
+        console.log(`Validation failed. Expected '${matchChecksum}' but got '${hash}'`);
+      }
+
+      resolve(match);
+    });
+  });
+}
+
+function download(config: GitDownloadOptions) {
+  return new Promise<void>((resolve, reject) => {
+    console.log(`Downloading Git from: ${config.source}`);
+
+    const options = {
+      headers: {
+        Accept: 'application/octet-stream',
+        'User-Agent': 'dugite'
+      },
+      secureProtocol: 'TLSv1_2_method'
+    };
+
+    const client = got.stream(config.source, options);
+
+    client.pipe(fs.createWriteStream(config.tempFile));
+
+    client.on('error', function (error) {
+      // @ts-ignore Property 'code' does not exist on type 'Error'.
+      if (error?.code === 'ETIMEDOUT') {
+        console.log(
+          `A timeout has occurred while downloading '${config.source}' - check ` +
+            `your internet connection and try again. If you are using a proxy, ` +
+            `make sure that the HTTP_PROXY and HTTPS_PROXY environment variables are set.`,
+          error
+        );
+      } else {
+        console.log(`Error raised while downloading ${config.source}`, error);
+      }
+      reject(error);
+    });
+
+    client.on('response', function (res) {
+      if (res.statusCode !== 200) {
+        console.log(`Non-200 response returned from ${config.source} - (${res.statusCode})`);
+        reject(`Non-200 response returned from ${config.source} - (${res.statusCode})`);
+      }
+
+      const len = parseInt(res.headers['content-length'], 10);
+
+      console.log();
+      const bar = new ProgressBar('Downloading Git [:bar] :percent :etas', {
+        complete: '=',
+        incomplete: ' ',
+        width: 50,
+        total: len
+      });
+
+      res.on('data', function (chunk) {
+        bar.tick(chunk.length);
+      });
+
+      res.on('end', function () {
+        resolve();
+      });
+    });
+  });
+}
+
+function unpackFile(file: fs.PathLike, outputPath: string) {
+  return new Promise<void>((resolve, reject) => {
+    function extract(source: fs.PathLike, callback: (err?: Error) => void) {
+      const extractor = tar
+        .extract({ cwd: outputPath })
+        .on('error', function (error: Error) {
+          callback(error);
+        })
+        .on('end', function () {
+          callback();
+        });
+
+      fs.createReadStream(source)
+        .on('error', function (error) {
+          callback(error);
+        })
+        // @ts-expect-error
+        .pipe(zlib.Gunzip())
+        .pipe(extractor);
+    }
+
+    console.log('Extracting file... (', file, ')');
+    extract(file, function (error) {
+      if (error) {
+        console.log('Unable to extract archive, aborting...', error);
+        return reject('Unable to extract archive, aborting... ' + error);
+      }
+
+      console.log('Done extracting file.');
+      resolve();
+    });
+  });
+}
+
+async function downloadAndUnpack(config: GitDownloadOptions) {
+  await download(config);
+
+  const valid = await verifyFile(config.tempFile, config.checksum);
+  if (valid) {
+    await unpackFile(config.tempFile, config.outputPath);
+  } else {
+    console.log(`checksum verification failed, refusing to unpack...`);
+    throw `checksum verification failed, refusing to unpack...`;
+  }
+}
+
+function mkdirpAsync(dir: string) {
+  return new Promise<void>((resolve, reject) => {
+    mkdirp(dir, function (error) {
+      if (error) {
+        return reject(error);
+      }
+
+      resolve();
+    });
+  });
+}
+
+export async function gitDownload(config: GitDownloadOptions) {
+  if (config.source === '') {
+    console.log(`Skipping downloading embedded Git as platform '${config.platform}' is not supported.`);
+    console.log(`To learn more about using dugite with a system Git: https://git.io/vF5oj`);
+    throw `Skipping downloading embedded Git as platform '${config.platform}' is not supported.`;
+  }
+
+  if (fs.existsSync(config.outputPath)) {
+    try {
+      console.log("Delete all old git files...")
+      rimraf.sync(config.outputPath);
+    } catch (err) {
+      console.error(err);
+    }
+  }
+
+  await mkdirpAsync(config.outputPath);
+
+  if (fs.existsSync(config.tempFile)) {
+    const valid = await verifyFile(config.tempFile, config.checksum);
+    if (valid) {
+      await unpackFile(config.tempFile, config.outputPath);
+    } else {
+      rimraf.sync(config.tempFile);
+      await downloadAndUnpack(config);
+    }
+  } else {
+    await downloadAndUnpack(config);
+  }
+}