refactor: data pipeline

.gitignore

@@ -1,2 +1,3 @@
 node_modules/
 /site/data/
+/.tmp/

README.md

@@ -10,6 +10,6 @@ This website indexes sf plugins based on npmjs.com and GitHub.
 
 ```
 yarn install
-GITHUB_TOKEN="$(gh auth token)" yarn build
+GITHUB_TOKEN="$(gh auth token)" yarn run crawl
 yarn develop
 ```

config.js

@@ -20,7 +20,7 @@ export const minPluginFields = [
 
 export const minCommandFields = ["pluginName", "id", "description", "link"];
 
-const excludeRules = [
+export const excludeRules = [
   {
     shouldExcludePackage: (pkg) => pkg.name === "sfdx-falcon-template",
     reason:
@@ -35,38 +35,3 @@ const excludeRules = [
     reason: "the GitHub repo is a fork",
   },
 ];
-
-function compareWithUndefined(a, b) {
-  if (a === b) {
-    return 0;
-  }
-  if (a === undefined) {
-    return -1;
-  }
-  if (b === undefined) {
-    return 1;
-  }
-  return a - b;
-}
-
-export function applyConfig(packages) {
-  return packages
-    .filter((pkg) => {
-      let keep = true;
-      for (const excludeRule of excludeRules) {
-        if (excludeRule.shouldExcludePackage(pkg)) {
-          console.error(
-            `ignoring package "${pkg.name}" because of "${excludeRule.reason}"`
-          );
-          keep = false;
-          break;
-        }
-      }
-      return keep;
-    })
-    .sort(
-      (a, b) =>
-        -1 *
-        compareWithUndefined(a?.gitHubStargazersCount, b?.gitHubStargazersCount)
-    );
-}

package.json

@@ -15,8 +15,9 @@
   },
   "scripts": {
     "build:vendor": "bash scripts/build.sh",
-    "build": "./scripts/get-plugins.js && ./scripts/add-github-repo-infos.js && ./scripts/get-commands.js && ./scripts/meta.js && ./scripts/minify.js",
-    "develop": "serve ./site",
+    "crawl": "./scripts/crawl.sh",
+    "build": "./scripts/aggregate.js",
+    "develop": "yarn run build && serve ./site",
     "start": "serve ./site"
   }
 }

scripts/aggregate.js

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { excludeRules, minCommandFields, minPluginFields } from "../config.js";
+import { applyConfig, getGitHubSlug } from "../utils.js";
+
+function pick(arr, fields) {
+  return arr.map((obj) =>
+    Object.fromEntries(
+      Object.entries(obj).filter(([key, _]) => fields.includes(key))
+    )
+  );
+}
+
+async function buildPlugins() {
+  const packages = JSON.parse(await readFile(join(".tmp", "packages.json")));
+  const dependencies = JSON.parse(
+    await readFile(join(".tmp", "package-dependencies.json"))
+  );
+  const repos = JSON.parse(await readFile(join(".tmp", "repos.json")));
+  const plugins = applyConfig(
+    packages.map((pkg) => {
+      return {
+        ...pkg,
+        ...dependencies.find((deps) => deps.name === pkg.name),
+        ...repos.find((repo) => repo.gitHubSlug === getGitHubSlug(pkg)),
+      };
+    }),
+    { excludeRules }
+  );
+  const minPlugins = pick(plugins, minPluginFields);
+  await writeFile(
+    join("site", "data", "plugins.min.json"),
+    JSON.stringify(minPlugins),
+    "utf8"
+  );
+}
+
+async function buildCommands() {
+  const commands = JSON.parse(await readFile(join(".tmp", "commands.json")));
+  const minCommands = pick(commands, minCommandFields);
+  await writeFile(
+    join("site", "data", "commands.min.json"),
+    JSON.stringify(minCommands),
+    "utf8"
+  );
+}
+
+async function buildMeta() {
+  const meta = {
+    lastUpdated: new Date().toISOString(),
+    source: "https://github.com/amtrack/sf-plugin-explorer",
+  };
+  await writeFile(
+    join("site", "data", "meta.json"),
+    JSON.stringify(meta),
+    "utf8"
+  );
+}
+
+async function main() {
+  await mkdir(join("site", "data"), { recursive: true });
+  await buildPlugins();
+  await buildCommands();
+  await buildMeta();
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exitCode = 1;
+});

scripts/crawl.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+./scripts/get-packages.js
+./scripts/get-package-dependencies.js
+./scripts/get-repos.js
+./scripts/get-commands.js

scripts/get-commands.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import pLimit from "p-limit";
 
@@ -45,15 +45,12 @@ async function getCommands(plugins) {
 }
 
 async function main() {
-  await mkdir(join("site", "data"), { recursive: true });
   const packageResults = JSON.parse(
-    await readFile(
-      join("site", "data", "packages-with-dependencies-and-github.json")
-    )
+    await readFile(join(".tmp", "packages.json"))
   );
   const commands = await getCommands(packageResults);
   await writeFile(
-    join("site", "data", "commands.json"),
+    join(".tmp", "commands.json"),
     JSON.stringify(commands),
     "utf8"
   );

scripts/get-package-dependencies.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import pLimit from "p-limit";
+
+async function getPackageDependencies(packageNames) {
+  const limit = pLimit(50);
+  const promises = packageNames.map((packageName) =>
+    limit(async () => {
+      const res = await fetch(
+        `https://registry.npmjs.org/${packageName}/latest`
+      );
+      return await res.json();
+    })
+  );
+  const packages = await Promise.all(promises);
+  return packages.map((pkg) => {
+    return {
+      name: pkg.name,
+      dependenciesCount: pkg.dependencies
+        ? Object.keys(pkg.dependencies).length
+        : 0,
+      pluginLibrary: getPluginLibrary(pkg),
+    };
+  });
+}
+
+function getPluginLibrary(pkg) {
+  if (pkg.dependencies?.["@salesforce/sf-plugins-core"]) {
+    return `@salesforce/sf-plugins-core@${pkg.dependencies?.["@salesforce/sf-plugins-core"]}`;
+  }
+  if (pkg.dependencies?.["@salesforce/command"]) {
+    return `@salesforce/command@${pkg.dependencies?.["@salesforce/command"]}`;
+  }
+  return "unknown";
+}
+
+async function main() {
+  const packages = JSON.parse(await readFile(join(".tmp", "packages.json")));
+  const npmPackageNames = packages.map((pkg) => pkg.name);
+  const dependencies = await getPackageDependencies(npmPackageNames);
+  await writeFile(
+    join(".tmp", "package-dependencies.json"),
+    JSON.stringify(dependencies),
+    "utf8"
+  );
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exitCode = 1;
+});

scripts/get-packages.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { npmSearchQuery } from "../config.js";
+
+async function searchNpmPackages(query, results = [], size = 250, page = 0) {
+  const from = size * page;
+  const res = await fetch(
+    `https://registry.npmjs.org/-/v1/search${query}&size=${size}&from=${from}`
+  );
+  const data = await res.json();
+  results.push(
+    ...data.objects.map((object) => {
+      return {
+        name: object.package.name,
+        version: object.package.version,
+        date: object.package.date,
+        description: object.package.description,
+        authorName: object.package.author?.name,
+        npmLink: object.package.links.npm,
+        gitHubLink: object.package.links.repository,
+        npmScoreFinal: object.score.final,
+      };
+    })
+  );
+  if (results.length < data.total) {
+    return await searchNpmPackages(query, results, size, page + 1);
+  }
+  return results;
+}
+
+async function main() {
+  const packages = await searchNpmPackages(npmSearchQuery);
+  await writeFile(
+    join(".tmp", "packages.json"),
+    JSON.stringify(packages),
+    "utf8"
+  );
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exitCode = 1;
+});