Skip to content

Commit

Permalink
Merge pull request #18 from dabigjoe6/refactor
Browse files Browse the repository at this point in the history 
Refactor
  • Loading branch information
@dabigjoe6
dabigjoe6 authored Jul 22, 2023
2 parents 74c5323 + abffe81 commit 18c0734
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 155 deletions.
16 changes: 9 additions & 7 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ import path from "path";
import { SQSEvent, Handler } from 'aws-lambda';
import { fileURLToPath } from "url";

import handleSubscription from "./subscription.js";
import handleSyncFeed from "./sync-feed.js";
import handleWelcomeEmail from "./welcome-email.js";

import SendFeed from "./send-feed.js";
import SyncFeed from "./sync-feed.js";
import Subscription from './subscription.js';

import sendWelcomeEmail from "./send-welcome-email.js";

const __filename = fileURLToPath(import.meta.url);

Expand All @@ -21,14 +21,16 @@ export const handler: Handler = async (event: SQSEvent) => {
console.log("Received message: ", message);

if (message[0] === 'subscription') {
await handleSubscription(message);
const subscription = new Subscription(message);
await subscription.init();
} else if (message[0] === 'sendfeed') {
const sendFeed = new SendFeed(message);
await sendFeed.init();
} else if (message[0] === 'syncfeed') {
await handleSyncFeed(message);
const syncFeed = new SyncFeed(message);
await syncFeed.init();
} else if (message[0] === 'welcomeemail') {
await handleWelcomeEmail(message);
await sendWelcomeEmail(message);
}

return {
Expand Down
4 changes: 2 additions & 2 deletions src/welcome-email.ts → src/send-welcome-email.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import welcomeEmailTemplate from "./utils/email/welcomeEmailTemplate.js";
const __filename = fileURLToPath(import.meta.url);
dotenv.config({ path: path.resolve(__filename, "../../.env") });

const handleWelcomeEmail = async (message: Array<string>) => {
const sendWelcomeEmail = async (message: Array<string>) => {
const email = message[1];

try {
Expand All @@ -29,4 +29,4 @@ const handleWelcomeEmail = async (message: Array<string>) => {
}
};

export default handleWelcomeEmail;
export default sendWelcomeEmail;
170 changes: 91 additions & 79 deletions src/subscription.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,66 +18,105 @@ const __filename = fileURLToPath(import.meta.url);

dotenv.config({ path: path.resolve(__filename, "../../.env") });

const BASE_URL = process.env.BASE_URL;

class Subscription {

const handleMediumService = async (authorId: string, url: string): Promise<Partial<ResourceI>[]> => {
const mediumScraper = new Medium();
console.log(
"Scraping medium for authorId: " + authorId + " from URL: " + url
);
return (await mediumScraper.getAllPosts(url)) || [];
};
authorId: string;
url: string;
service: Sources;

const handleSubstackService = async (authorId: string, url: string): Promise<Partial<ResourceI>[]> => {
const substackScraper = new Substack();
console.log(
"Scraping substack for authorId: " + authorId + " from URL: " + url
);
return (await substackScraper.getAllPosts(url)) || [];
};
BASE_URL = process.env.BASE_URL;

const handleRSSService = async (authorId: string, url: string): Promise<Partial<ResourceI>[]> => {
const rssScraper = new RSS();
console.log(
"Scraping RSS feed for authorId: " + authorId + " from URL: " + url
);
return (await rssScraper.getAllPosts(url)) || [];
}

const getPostsFromService = async (service: Sources, authorId: string, url: string) => {
let posts: Partial<ResourceI>[] = [];
switch (service) {
case Sources.MEDIUM:
posts = await handleMediumService(authorId, url)
break;
case Sources.SUBSTACK:
posts = await handleSubstackService(authorId, url);
break;
case Sources.RSS:
posts = await handleRSSService(authorId, url);
break;
default:
posts = [];
throw new Error("Service not supported: " + service);
constructor(message: Array<string>) {
this.authorId = (message[1] as unknown) as string;

if (!this.authorId) {
throw new Error(
"Could not get author Id - subscriptionConsumer.ts"
);
}
this.url = message[2];
if (!this.url) {
throw new Error(
"Could not get authors url - subscriptionConsumer.ts"
);
}
this.service = (message[3] as unknown) as Sources;
if (!this.service) {
throw new Error(
"Could not determine service - subscriptionConsumer.ts"
);
}
}

if (posts && isArray(posts) && posts.length > 0) {
posts = posts.map((post) => ({
...post,
source: service,
author: authorId,
}));
private async handleMediumService(): Promise<Partial<ResourceI>[]> {
const mediumScraper = new Medium();
console.log(
"Scraping medium for authorId: " + this.authorId + " from URL: " + this.url
);
return (await mediumScraper.getAllPosts(this.url)) || [];
};

private async handleSubstackService(): Promise<Partial<ResourceI>[]> {
const substackScraper = new Substack();
console.log(
"Scraping substack for authorId: " + this.authorId + " from URL: " + this.url
);
return (await substackScraper.getAllPosts(this.url)) || [];
};

private async handleRSSService(): Promise<Partial<ResourceI>[]> {
const rssScraper = new RSS();
console.log(
"Scraping RSS feed for authorId: " + this.authorId + " from URL: " + this.url
);
return (await rssScraper.getAllPosts(this.url)) || [];
}

private async getPostsFromService() {
let posts: Partial<ResourceI>[] = [];
switch (this.service) {
case Sources.MEDIUM:
posts = await this.handleMediumService()
break;
case Sources.SUBSTACK:
posts = await this.handleSubstackService();
break;
case Sources.RSS:
posts = await this.handleRSSService();
break;
default:
posts = [];
throw new Error("Service not supported: " + this.service);
}

if (posts && isArray(posts) && posts.length > 0) {
posts = posts.map((post) => ({
...post,
source: this.service,
author: this.authorId,
}));
} else if (posts.length === 0) {
console.warn("Empty posts");
} else if (!isArray(posts)) {
console.warn("Posts not an array, expecting an array");
} else {
console.warn("Posts is " + posts);
}

return posts
}

private async savePostsToDB(posts: Partial<ResourceI>[]) {
try {
console.log("Saving posts to DB");
//Update Resource collection with crawled articles
const response = await fetch(BASE_URL + "/subscribe/saveAuthorsPosts", {
const response = await fetch(this.BASE_URL + "/subscribe/saveAuthorsPosts", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ posts, source: service }),
body: JSON.stringify({ posts, source: this.service }),
});

const data: { message?: string } = (await response.json()) || { message: '' };
Expand All @@ -91,41 +130,14 @@ const getPostsFromService = async (service: Sources, authorId: string, url: stri

console.log("Succesfully saved posts!", data.message);
} catch (err) {
console.error(err);
console.error("Could not save posts to DB", err);
}

} else if (posts.length === 0) {
console.warn("Empty posts");
} else if (!isArray(posts)) {
console.warn("Posts not an array, expecting an array");
} else {
console.warn("Posts is " + posts);
}
};


const handleSubscription = async (message: Array<string>) => {
const authorId = (message[1] as unknown) as string;

if (!authorId) {
throw new Error(
"Could not get author Id - subscriptionConsumer.ts"
);
}
const url = message[2];
if (!url) {
throw new Error(
"Could not get authors url - subscriptionConsumer.ts"
);
}
const service = (message[3] as unknown) as Sources;
if (!service) {
throw new Error(
"Could not determine service - subscriptionConsumer.ts"
);
}
async init() {
await this.savePostsToDB(await this.getPostsFromService())
};

await getPostsFromService(service, authorId, url);
}
};

export default handleSubscription;
export default Subscription;
Loading

0 comments on commit 18c0734

Please sign in to comment.