diff --git a/src/index.ts b/src/index.ts index a7a84e5..b9d2a41 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,11 +3,11 @@ import path from "path"; import { SQSEvent, Handler } from 'aws-lambda'; import { fileURLToPath } from "url"; -import handleSubscription from "./subscription.js"; -import handleSyncFeed from "./sync-feed.js"; -import handleWelcomeEmail from "./welcome-email.js"; - import SendFeed from "./send-feed.js"; +import SyncFeed from "./sync-feed.js"; +import Subscription from './subscription.js'; + +import sendWelcomeEmail from "./send-welcome-email.js"; const __filename = fileURLToPath(import.meta.url); @@ -21,14 +21,16 @@ export const handler: Handler = async (event: SQSEvent) => { console.log("Received message: ", message); if (message[0] === 'subscription') { - await handleSubscription(message); + const subscription = new Subscription(message); + await subscription.init(); } else if (message[0] === 'sendfeed') { const sendFeed = new SendFeed(message); await sendFeed.init(); } else if (message[0] === 'syncfeed') { - await handleSyncFeed(message); + const syncFeed = new SyncFeed(message); + await syncFeed.init(); } else if (message[0] === 'welcomeemail') { - await handleWelcomeEmail(message); + await sendWelcomeEmail(message); } return { diff --git a/src/welcome-email.ts b/src/send-welcome-email.ts similarity index 90% rename from src/welcome-email.ts rename to src/send-welcome-email.ts index 3ec2bde..a067cb5 100644 --- a/src/welcome-email.ts +++ b/src/send-welcome-email.ts @@ -8,7 +8,7 @@ import welcomeEmailTemplate from "./utils/email/welcomeEmailTemplate.js"; const __filename = fileURLToPath(import.meta.url); dotenv.config({ path: path.resolve(__filename, "../../.env") }); -const handleWelcomeEmail = async (message: Array) => { +const sendWelcomeEmail = async (message: Array) => { const email = message[1]; try { @@ -29,4 +29,4 @@ const handleWelcomeEmail = async (message: Array) => { } }; -export default handleWelcomeEmail; \ No newline at end of file +export default sendWelcomeEmail; \ No newline at end of file diff --git a/src/subscription.ts b/src/subscription.ts index 7a3a523..4686a4f 100644 --- a/src/subscription.ts +++ b/src/subscription.ts @@ -18,66 +18,105 @@ const __filename = fileURLToPath(import.meta.url); dotenv.config({ path: path.resolve(__filename, "../../.env") }); -const BASE_URL = process.env.BASE_URL; +class Subscription { -const handleMediumService = async (authorId: string, url: string): Promise[]> => { - const mediumScraper = new Medium(); - console.log( - "Scraping medium for authorId: " + authorId + " from URL: " + url - ); - return (await mediumScraper.getAllPosts(url)) || []; -}; + authorId: string; + url: string; + service: Sources; -const handleSubstackService = async (authorId: string, url: string): Promise[]> => { - const substackScraper = new Substack(); - console.log( - "Scraping substack for authorId: " + authorId + " from URL: " + url - ); - return (await substackScraper.getAllPosts(url)) || []; -}; + BASE_URL = process.env.BASE_URL; -const handleRSSService = async (authorId: string, url: string): Promise[]> => { - const rssScraper = new RSS(); - console.log( - "Scraping RSS feed for authorId: " + authorId + " from URL: " + url - ); - return (await rssScraper.getAllPosts(url)) || []; -} - -const getPostsFromService = async (service: Sources, authorId: string, url: string) => { - let posts: Partial[] = []; - switch (service) { - case Sources.MEDIUM: - posts = await handleMediumService(authorId, url) - break; - case Sources.SUBSTACK: - posts = await handleSubstackService(authorId, url); - break; - case Sources.RSS: - posts = await handleRSSService(authorId, url); - break; - default: - posts = []; - throw new Error("Service not supported: " + service); + constructor(message: Array) { + this.authorId = (message[1] as unknown) as string; + + if (!this.authorId) { + throw new Error( + "Could not get author Id - subscriptionConsumer.ts" + ); + } + this.url = message[2]; + if (!this.url) { + throw new Error( + "Could not get authors url - subscriptionConsumer.ts" + ); + } + this.service = (message[3] as unknown) as Sources; + if (!this.service) { + throw new Error( + "Could not determine service - subscriptionConsumer.ts" + ); + } } - if (posts && isArray(posts) && posts.length > 0) { - posts = posts.map((post) => ({ - ...post, - source: service, - author: authorId, - })); + private async handleMediumService(): Promise[]> { + const mediumScraper = new Medium(); + console.log( + "Scraping medium for authorId: " + this.authorId + " from URL: " + this.url + ); + return (await mediumScraper.getAllPosts(this.url)) || []; + }; + private async handleSubstackService(): Promise[]> { + const substackScraper = new Substack(); + console.log( + "Scraping substack for authorId: " + this.authorId + " from URL: " + this.url + ); + return (await substackScraper.getAllPosts(this.url)) || []; + }; + + private async handleRSSService(): Promise[]> { + const rssScraper = new RSS(); + console.log( + "Scraping RSS feed for authorId: " + this.authorId + " from URL: " + this.url + ); + return (await rssScraper.getAllPosts(this.url)) || []; + } + + private async getPostsFromService() { + let posts: Partial[] = []; + switch (this.service) { + case Sources.MEDIUM: + posts = await this.handleMediumService() + break; + case Sources.SUBSTACK: + posts = await this.handleSubstackService(); + break; + case Sources.RSS: + posts = await this.handleRSSService(); + break; + default: + posts = []; + throw new Error("Service not supported: " + this.service); + } + + if (posts && isArray(posts) && posts.length > 0) { + posts = posts.map((post) => ({ + ...post, + source: this.service, + author: this.authorId, + })); + } else if (posts.length === 0) { + console.warn("Empty posts"); + } else if (!isArray(posts)) { + console.warn("Posts not an array, expecting an array"); + } else { + console.warn("Posts is " + posts); + } + + return posts + } + + private async savePostsToDB(posts: Partial[]) { try { console.log("Saving posts to DB"); //Update Resource collection with crawled articles - const response = await fetch(BASE_URL + "/subscribe/saveAuthorsPosts", { + const response = await fetch(this.BASE_URL + "/subscribe/saveAuthorsPosts", { method: "POST", headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ posts, source: service }), + body: JSON.stringify({ posts, source: this.service }), }); const data: { message?: string } = (await response.json()) || { message: '' }; @@ -91,41 +130,14 @@ const getPostsFromService = async (service: Sources, authorId: string, url: stri console.log("Succesfully saved posts!", data.message); } catch (err) { - console.error(err); + console.error("Could not save posts to DB", err); } - - } else if (posts.length === 0) { - console.warn("Empty posts"); - } else if (!isArray(posts)) { - console.warn("Posts not an array, expecting an array"); - } else { - console.warn("Posts is " + posts); } -}; - - -const handleSubscription = async (message: Array) => { - const authorId = (message[1] as unknown) as string; - if (!authorId) { - throw new Error( - "Could not get author Id - subscriptionConsumer.ts" - ); - } - const url = message[2]; - if (!url) { - throw new Error( - "Could not get authors url - subscriptionConsumer.ts" - ); - } - const service = (message[3] as unknown) as Sources; - if (!service) { - throw new Error( - "Could not determine service - subscriptionConsumer.ts" - ); - } + async init() { + await this.savePostsToDB(await this.getPostsFromService()) + }; - await getPostsFromService(service, authorId, url); -} +}; -export default handleSubscription; \ No newline at end of file +export default Subscription; \ No newline at end of file diff --git a/src/sync-feed.ts b/src/sync-feed.ts index a8d8cee..1c75282 100644 --- a/src/sync-feed.ts +++ b/src/sync-feed.ts @@ -17,82 +17,92 @@ const __filename = fileURLToPath(import.meta.url); dotenv.config({ path: path.resolve(__filename, "../../.env") }); -const BASE_URL = process.env.BASE_URL; -const syncAuthorsResources = async (posts: Array, authorId: string) => { +class SyncFeed { - const response = await fetch(BASE_URL + "/resource/sync-resources", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ posts, authorId }), - }); + authorId: string; + url: string; + source: Sources; + newPosts: ResourceI[]; - const data: { message?: string } = (await response.json()) || { message: '' }; + BASE_URL = process.env.BASE_URL; - if (!response.ok) { - if (response.status === 404) throw new Error("Not found: " + data?.message || ""); - else if (response.status === 401) - throw new Error("Unauthorized: " + data?.message); - else throw new Error(data?.message); + constructor(message: Array) { + this.authorId = (message[1] as unknown) as string; + this.url = (message[2] as unknown) as string; + this.source = (message[3] as unknown) as Sources; } - console.log("Succesfully saved posts!", data.message); -} - - -const syncPosts = async (newPosts: ResourceI[], service: Sources, authorId: string) => { - if (!newPosts) { - console.error( - "newPosts is undefined but required - syncResourcesConsumer" - ); - return; + private getScraperInstance(source: Sources) { + switch (source) { + case Sources.MEDIUM: + return new Medium(); + case Sources.SUBSTACK: + return new Substack(); + case Sources.RSS: + return new RSS(); + default: + throw new Error("Not a valid source - syncResourcesConsumer"); + } + }; + + private async syncAuthorsResources(posts: Array, authorId: string) { + const response = await fetch(this.BASE_URL + "/resource/sync-resources", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ posts, authorId }), + }); + + const data: { message?: string } = (await response.json()) || { message: '' }; + + if (!response.ok) { + if (response.status === 404) throw new Error("Not found: " + data?.message || ""); + else if (response.status === 401) + throw new Error("Unauthorized: " + data?.message); + else throw new Error(data?.message); + } + + console.log("Succesfully saved posts!", data.message); } - if (isArray(newPosts) && !newPosts.length) { - console.warn("No new posts - syncResourcesConsumer"); - return; - } - let posts = newPosts.map((post) => ({ - ...post, - source: service, - author: authorId, - latest: true - })); - - console.log("Saving posts to DB"); - //Update Resource collection with new posts - await syncAuthorsResources(posts, authorId); - return; -}; - - -const getScraperInstance = (source: Sources) => { - switch (source) { - case Sources.MEDIUM: - return new Medium(); - case Sources.SUBSTACK: - return new Substack(); - case Sources.RSS: - return new RSS(); - default: - throw new Error("Not a valid source - syncResourcesConsumer"); + private async syncPosts(newPosts: ResourceI[], service: Sources, authorId: string) { + if (!newPosts) { + console.error( + "newPosts is undefined but required - syncResourcesConsumer" + ); + return; + } + + if (isArray(newPosts) && !newPosts.length) { + console.warn("No new posts - syncResourcesConsumer"); + return; + } + + let posts = newPosts.map((post) => ({ + ...post, + source: service, + author: authorId, + latest: true + })); + + console.log("Saving posts to DB"); + //Update Resource collection with new posts + await this.syncAuthorsResources(posts, authorId); + return; + }; + + async init() { + try { + const scraperInstance = this.getScraperInstance(this.source); + this.newPosts = (await scraperInstance.getAllPosts(this.url)) as ResourceI[]; + await this.syncPosts(this.newPosts, this.source, this.authorId) + } catch (err) { + console.error("Could not sync feed: ", err); + } } -}; - -const handleSyncFeed = async (message: Array) => { - const authorId = (message[1] as unknown) as string; - const url = (message[2] as unknown) as string; - const source = (message[3] as unknown) as Sources; - - - const scraperInstance = getScraperInstance(source); - - const newPosts = (await scraperInstance.getAllPosts(url)) as ResourceI[]; - - await syncPosts(newPosts, source, authorId) } -export default handleSyncFeed; \ No newline at end of file +export default SyncFeed; \ No newline at end of file