init

.gitignore

@@ -0,0 +1,3 @@
+**.env
+node_modules
+coverage

README.md

@@ -0,0 +1,66 @@
+# Docs Assertion Tester
+
+Do you love unit tests for your code? Do you wish you could write unit tests for your documentation? Well, now you can!
+
+<!-- TODO: Add gif -->
+
+## Prerequisites
+
+- An OpenAI API key
+- A GitHub token with `repo` scope
+
+We recommend storing these in GitHub secrets. You can find instructions on how to do that
+[here](https://docs.github.com/en/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository).
+
+## Installation
+
+In any workflow for PRs, add the following step:
+
+```yaml
+- name: Docs Assertion Tester
+  uses: hasura/docs-assertion-tester@v1
+  with:
+    openai_api_key: ${{ secrets.OPENAI_API_KEY }}
+    github_token: ${{ secrets.GITHUB_TOKEN }}
+    org: ${{ github.repository_owner }}
+    repo: ${{ github.repository }}
+    pr_number: ${{ github.event.number }}
+```
+
+## Usage
+
+In the description of any PR, add the following comments:
+
+```html
+<!-- DX:Assertion-start -->
+<!-- DX:Assertion-end -->
+```
+
+Then, between the start and end comments, add your assertions in the following format:
+
+```html
+<!-- DX:Assertion-start -->
+A user should be able to easily add a comment to their PR's description.
+<!-- DX:Assertion-end -->
+```
+
+The assertion tester will then run your assertions against the documentation in your PR's description. It will check
+over two scopes:
+
+- `Diff`
+- `Integrated`
+
+The `Diff` scope will check the assertions against the diff (i.e., only what the author contributed). The `Integrated`
+scope will check the assertions against the entire set of files changed, including the author's changes.
+
+Upon completion, the assertion tester will output the analysis in markdown format. You can add a comment to your PR
+using our handy [GitHub Action](https://github.com/marketplace/actions/comment-progress).
+
+Using our `comment-progress` action, the output looks like this after running [the sample workflow](#) in the `/samples`
+folder:
+
+<!-- TODO: Add screenshot -->
+
+## Contributing
+
+Before opening a PR, please create an issue to discuss the proposed change.

__tests__/github.test.ts

@@ -0,0 +1,50 @@
+import {
+  testConnection,
+  getRepo,
+  getPullRequests,
+  getSinglePR,
+  getAssertion,
+  getDiff,
+  getChangedFiles,
+  getFileContent,
+} from '../src/github';
+
+describe('GitHub Functionality', () => {
+  it('Should be able to connect to GitHub', () => {
+    expect(testConnection()).resolves.toBe(true);
+  });
+  it('Should be able to access a repo in the Hasura org', () => {
+    expect(getRepo('hasura', 'v3-docs')).resolves.toHaveProperty('name', 'v3-docs');
+  });
+  it('Should be able to get the PRs for the repo', async () => {
+    const pullRequests = await getPullRequests('hasura', 'v3-docs');
+    expect(pullRequests?.length).toBeGreaterThanOrEqual(0);
+  });
+  it('Should be able to get the contents of a single PR', () => {
+    const prNumber: number = 262;
+    expect(getSinglePR('hasura', 'v3-docs', prNumber)).resolves.toHaveProperty('number', prNumber);
+  });
+  it('Should be able to get the assertion from the description of a PR', async () => {
+    // test PR with 262 about PATs
+    const prNumber: number = 262;
+    const PR = await getSinglePR('hasura', 'v3-docs', prNumber);
+    const assertion = await getAssertion(PR?.body || '');
+    // the last I checked, this was the text — if it's failing, check th PR 🤷‍♂️
+    expect(assertion).toContain('understand how to log in using the PAT with VS Code.');
+  });
+  it('Should be able to return the diff of a PR', async () => {
+    const diffUrl: number = 262;
+    const diff = await getDiff(diffUrl);
+    expect(diff).toContain('diff');
+  });
+  it('Should be able to determine which files were changed in a PR', async () => {
+    const diffUrl: number = 262;
+    const diff = await getDiff(diffUrl);
+    const files = getChangedFiles(diff);
+    expect(files).toContain('docs/ci-cd/projects.mdx');
+  });
+  it('Should be able to get the contents of a file', async () => {
+    const contents = await getFileContent(['README.md']);
+    expect(contents).toContain('Website');
+  });
+});

__tests__/openai.test.ts

@@ -0,0 +1,39 @@
+import { testConnection, generatePrompt, testAssertion, writeAnalysis } from '../src/open_ai';
+import { getDiff, getSinglePR, getAssertion, getChangedFiles, getFileContent } from '../src/github';
+
+describe('OpenAI Functionality', () => {
+  it('Should be able to connect to OpenAI', async () => {
+    await expect(testConnection()).resolves.toBe(true);
+  });
+  it('Should be able to generate a prompt using the diff, assertion, and whole file', async () => {
+    const diff: string = await getDiff(262);
+    const assertion: string = 'The documentation is easy to read and understand.';
+    const file: string = 'This is a test file.';
+    const prompt: string = generatePrompt(diff, assertion, file);
+    expect(prompt).toContain(assertion);
+    expect(prompt).toContain(diff);
+    expect(prompt).toContain(file);
+  });
+  it.todo('Should return null when an error is thrown');
+  it('Should be able to generate a response using the prompt and a sample diff', async () => {
+    // This should produce somewhat regularly happy results 👇
+    // const prNumber: number = 243;
+    // This should produce somewhat regularly unhappy results 👇
+    const prNumber: number = 262;
+    const PR = await getSinglePR('hasura', 'v3-docs', prNumber);
+    const assertion = await getAssertion(PR?.body || '');
+    const diff: string = await getDiff(prNumber);
+    const changedFiles = getChangedFiles(diff);
+    const file: any = await getFileContent(changedFiles);
+    const prompt: string = generatePrompt(diff, assertion, file);
+    const response = await testAssertion(prompt);
+    expect(response).toBeTruthy();
+  }, 50000);
+  it('Should create a nicely formatted message using the response', async () => {
+    expect(
+      writeAnalysis(
+        `[{"satisfied": "\u2705", "scope": "diff", "feedback": "You did a great job!"}, {"satisfied": "\u2705", "scope": "wholeFile", "feedback": "Look. At. You. Go!"}]`
+      )
+    ).toContain('You did a great job!');
+  });
+});

action.yml

@@ -0,0 +1,27 @@
+name: 'Docs Assertion Tester'
+description: 'Test user-facing assertions about documentation changes using OpenAI.'
+branding:
+  icon: user-check
+  color: white
+inputs:
+  GITHUB_TOKEN:
+    description: 'GitHub token'
+    required: true
+  OPENAI_API_KEY:
+    description: 'OpenAI API key'
+    required: true
+  GITHUB_ORG:
+    description: 'The owner of the GitHub repository'
+    required: true
+  GITHUB_REPOSITORY:
+    description: 'The name of the GitHub repository'
+    required: true
+  PR_NUMBER:
+    description: 'Pull Request number'
+    required: true
+outputs:
+  analysis:
+    description: 'The analysis of the PR from OpenAI.'
+runs:
+  using: 'node18'
+  main: 'dist/index.js'

bable.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
+};

dist/github/config.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.github = void 0;
+const rest_1 = require("@octokit/rest");
+exports.github = new rest_1.Octokit({
+    auth: process.env.GITHUB_TOKEN,
+});
+console.log(exports.github);

dist/github/index.js

@@ -0,0 +1,150 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFileContent = exports.getChangedFiles = exports.getDiff = exports.getAssertion = exports.getSinglePR = exports.getPullRequests = exports.getRepo = exports.testConnection = exports.github = void 0;
+const dotenv_1 = __importDefault(require("dotenv"));
+const core = __importStar(require("@actions/core"));
+const rest_1 = require("@octokit/rest");
+dotenv_1.default.config();
+const token = core.getInput('GITHUB_TOKEN') || process.env.GITHUB_TOKEN;
+exports.github = new rest_1.Octokit({
+    auth: token,
+});
+// We'll use this to test a connection to GitHub
+const testConnection = async () => {
+    try {
+        const { data } = await exports.github.repos.listForAuthenticatedUser();
+        console.log(`🚀 Connection to GH established`);
+        return true;
+    }
+    catch (error) {
+        console.error(error);
+        return false;
+    }
+};
+exports.testConnection = testConnection;
+// If we can get a connection, we can get access to a repo, or we'll return null because
+const getRepo = async (owner, repo) => {
+    try {
+        const { data } = await exports.github.repos.get({ owner, repo });
+        console.log(`🐕 Fetched the repo`);
+        return data;
+    }
+    catch (error) {
+        console.error(error);
+        return null;
+    }
+};
+exports.getRepo = getRepo;
+// If we can get a repo, we can get all the PRs associated with it
+const getPullRequests = async (owner, repo) => {
+    try {
+        const { data } = await exports.github.pulls.list({ owner, repo });
+        console.log(`🐕 Fetched all PRs`);
+        return data;
+    }
+    catch (error) {
+        console.error(error);
+        return null;
+    }
+};
+exports.getPullRequests = getPullRequests;
+// We should be able to get a PR by its number
+const getSinglePR = async (owner, repo, prNumber) => {
+    try {
+        const { data } = await exports.github.pulls.get({ owner, repo, pull_number: prNumber });
+        console.log(`✅ Got PR #${prNumber}`);
+        return data;
+    }
+    catch (error) {
+        console.error(error);
+        return null;
+    }
+};
+exports.getSinglePR = getSinglePR;
+// If we can get a PR, we can parse the description and isolate the assertion using the comments
+const getAssertion = async (description) => {
+    //   find everything in between <!-- DX:Assertion-start --> and <!-- DX:Assertion-end -->
+    const regex = /<!-- DX:Assertion-start -->([\s\S]*?)<!-- DX:Assertion-end -->/g;
+    const assertion = regex.exec(description);
+    if (assertion) {
+        console.log(`✅ Got assertion: ${assertion[1]}`);
+        return assertion[1];
+    }
+    return null;
+};
+exports.getAssertion = getAssertion;
+// If we have a diff_url we can get the diff
+const getDiff = async (prNumber) => {
+    const { data: diff } = await exports.github.pulls.get({
+        owner: 'hasura',
+        repo: 'v3-docs',
+        pull_number: prNumber,
+        mediaType: {
+            format: 'diff',
+        },
+    });
+    // We'll have to convert the diff to a string, then we can return it
+    const diffString = diff.toString();
+    console.log(`✅ Got diff for PR #${prNumber}`);
+    return diffString;
+};
+exports.getDiff = getDiff;
+// If we have the diff, we can determine which files were changed
+const getChangedFiles = (diff) => {
+    const fileLines = diff.split('\n').filter((line) => line.startsWith('diff --git'));
+    const changedFiles = fileLines
+        .map((line) => {
+        const paths = line.split(' ').slice(2);
+        return paths.map((path) => path.replace('a/', '').replace('b/', ''));
+    })
+        .flat();
+    console.log(`✅ Found ${changedFiles.length} affected files`);
+    return [...new Set(changedFiles)];
+};
+exports.getChangedFiles = getChangedFiles;
+// We'll also need to get the whole file using the files changed from
+async function getFileContent(path) {
+    let content = '';
+    // loop over the array of files
+    for (let i = 0; i < path.length; i++) {
+        // get the file content
+        const { data } = await exports.github.repos.getContent({
+            owner: 'hasura',
+            repo: 'v3-docs',
+            path: path[i],
+        });
+        // decode the file content
+        const decodedContent = Buffer.from(data.content, 'base64').toString();
+        // add the decoded content to the content string
+        content += decodedContent;
+    }
+    console.log(`✅ Got file(s) contents`);
+    return content;
+}
+exports.getFileContent = getFileContent;