Merge pull request #320 from ekalinin/simple-sitemap

simple sitemap, bugfixes

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 6.2.0
+
+- Add simplified interface for creating sitemaps and index
+- fix bug where sitemap and index stream would not properly wait to emit finish event until all sitemaps had been written
+- bump deps
+
 ## 6.1.7
 
 - Improve documentation and error messaging on ending a stream too early #317

README.md

@@ -106,6 +106,33 @@ app.listen(3000, () => {
 
 If you know you are definitely going to have more than 50,000 urls in your sitemap, you can use this slightly more complex interface to create a new sitemap every 45,000 entries and add that file to a sitemap index.
 
+```js
+const { createReadStream, createWriteStream } = require('fs');
+const { resolve } = require('path');
+const { createGzip } = require('zlib')
+const {
+  simpleSitemapAndIndex,
+  lineSeparatedURLsToSitemapOptions
+} = require('sitemap')
+
+// writes sitemaps and index out to the destination you provide
+simpleSitemapAndIndex({
+  hostname: 'https://example.com',
+  destinationDir: './',
+  sourceData: lineSeparatedURLsToSitemapOptions(
+    createReadStream('./your-data.json.txt')
+  ),
+  // or
+  sourceData: [{ url: '/page-1/', changefreq: 'daily'}, ...],
+  // or
+  sourceData: './your-data.json.txt',
+}).then(() => {
+  // Do follow up actions
+})
+```
+
+Want to customize that?
+
 ```js
 const { createReadStream, createWriteStream } = require('fs');
 const { resolve } = require('path');
@@ -117,7 +144,7 @@ const {
 } = require('sitemap')
 
 const sms = new SitemapAndIndexStream({
-  limit: 10000, // defaults to 45k
+  limit: 50000, // defaults to 45k
   // SitemapAndIndexStream will call this user provided function every time
   // it needs to create a new sitemap file. You merely need to return a stream
   // for it to write the sitemap urls to and the expected url where that sitemap will be hosted

index.ts

@@ -38,3 +38,5 @@ export {
   ObjectStreamToJSON,
   ObjectStreamToJSONOptions,
 } from './lib/sitemap-parser';
+
+export { simpleSitemapAndIndex } from './lib/sitemap-simple';

lib/sitemap-index-stream.ts

@@ -190,22 +190,22 @@ export class SitemapAndIndexStream extends SitemapIndexStream {
       this._writeSMI(item);
       super._transform(this.idxItem, encoding, callback);
     } else if (this.i % this.limit === 0) {
-      this.currentSitemap.end();
-      const [idxItem, currentSitemap] = this.getSitemapStream(
-        this.i / this.limit
-      );
-      this.currentSitemap = currentSitemap;
-      this._writeSMI(item);
-      // push to index stream
-      super._transform(idxItem, encoding, callback);
+      this.currentSitemap.end(() => {
+        const [idxItem, currentSitemap] = this.getSitemapStream(
+          this.i / this.limit
+        );
+        this.currentSitemap = currentSitemap;
+        this._writeSMI(item);
+        // push to index stream
+        super._transform(idxItem, encoding, callback);
+      });
     } else {
       this._writeSMI(item);
       callback();
     }
   }
 
   _flush(cb: TransformCallback): void {
-    this.currentSitemap.end();
-    super._flush(cb);
+    this.currentSitemap.end(() => super._flush(cb));
   }
 }

lib/sitemap-simple.ts

@@ -0,0 +1,66 @@
+import {
+  SitemapAndIndexStream,
+  SitemapStream,
+  lineSeparatedURLsToSitemapOptions,
+} from '../index';
+import { createGzip } from 'zlib';
+import { createWriteStream, createReadStream } from 'fs';
+import { resolve } from 'path';
+import { Readable, pipeline as pline } from 'stream';
+import { SitemapItemLoose } from './types';
+import { promisify } from 'util';
+import { URL } from 'url';
+
+const pipeline = promisify(pline);
+export const simpleSitemapAndIndex = ({
+  hostname,
+  sitemapHostname = hostname, // if different
+  /**
+   * Pass a line separated list of sitemap items or a stream or an array
+   */
+  sourceData,
+  destinationDir,
+  limit = 50000,
+}: {
+  hostname: string;
+  sitemapHostname?: string;
+  sourceData: SitemapItemLoose | string | Readable | string[];
+  destinationDir: string;
+  limit?: number;
+}): Promise<void> => {
+  const sitemapAndIndexStream = new SitemapAndIndexStream({
+    limit,
+    getSitemapStream: (i) => {
+      const sitemapStream = new SitemapStream({
+        hostname,
+      });
+      const path = `./sitemap-${i}.xml`;
+
+      sitemapStream
+        .pipe(createGzip()) // compress the output of the sitemap
+        .pipe(createWriteStream(resolve(destinationDir, path + '.gz'))); // write it to sitemap-NUMBER.xml
+
+      return [new URL(path, sitemapHostname).toString(), sitemapStream];
+    },
+  });
+  let src: Readable;
+  if (typeof sourceData === 'string') {
+    src = lineSeparatedURLsToSitemapOptions(createReadStream(sourceData));
+  } else if (sourceData instanceof Readable) {
+    src = sourceData;
+  } else if (Array.isArray(sourceData)) {
+    src = Readable.from(sourceData);
+  } else {
+    throw new Error(
+      "unhandled source type. You've passed in data that is not supported"
+    );
+  }
+  return pipeline(
+    src,
+    sitemapAndIndexStream,
+    createGzip(),
+    createWriteStream(resolve(destinationDir, './sitemap-index.xml.gz'))
+  );
+};
+
+export default simpleSitemapAndIndex;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sitemap",
-  "version": "6.1.7",
+  "version": "6.2.0",
   "description": "Sitemap-generating lib/cli",
   "keywords": [
     "sitemap",
@@ -149,7 +149,7 @@
     }
   },
   "dependencies": {
-    "@types/node": "^14.0.14",
+    "@types/node": "^14.0.18",
     "@types/sax": "^1.2.1",
     "arg": "^4.1.3",
     "sax": "^1.2.4"
@@ -162,15 +162,15 @@
     "@babel/plugin-transform-typescript": "^7.10.4",
     "@babel/preset-env": "^7.10.4",
     "@babel/preset-typescript": "^7.10.4",
-    "@types/jest": "^26.0.3",
-    "@typescript-eslint/eslint-plugin": "^3.5.0",
-    "@typescript-eslint/parser": "^3.5.0",
+    "@types/jest": "^26.0.4",
+    "@typescript-eslint/eslint-plugin": "^3.6.0",
+    "@typescript-eslint/parser": "^3.6.0",
     "babel-eslint": "^10.1.0",
     "babel-polyfill": "^6.26.0",
     "concurrently": "^5.2.0",
-    "eslint": "^7.3.1",
+    "eslint": "^7.4.0",
     "eslint-config-prettier": "^6.11.0",
-    "eslint-plugin-jest": "^23.17.1",
+    "eslint-plugin-jest": "^23.18.0",
     "eslint-plugin-prettier": "^3.1.4",
     "express": "^4.17.1",
     "husky": "^4.2.5",
@@ -182,7 +182,7 @@
     "stats-lite": "^2.2.0",
     "stream-json": "^1.5.0",
     "through2-map": "^3.0.0",
-    "typescript": "^3.9.5"
+    "typescript": "^3.9.6"
   },
   "engines": {
     "node": ">=10.3.0",