cleaning up GCS

Job will run daily at 9am to fetch latest files then merge them and save locally into the repo

.github/workflows/fetchLatestData.yml

@@ -2,8 +2,8 @@ name: Generate Latest Data
 
 on:
   schedule:
-    # Runs at 8 AM UTC every day
-    - cron: '0 8 * * *'
+    # Runs at 9 AM UTC every day
+    - cron: '0 9 * * *'
   workflow_dispatch:
 
 jobs:

utils/GenerateLatestData.js

@@ -1,17 +1,5 @@
-// import { Storage } from '@google-cloud/storage';
-// import * as fs from 'node:fs/promises';
-
 // Define the regions to search for files
 const regions = ['cdg', 'sea', 'iad'];
-const bucketName = 'thefastest-data'; // The name of your GCS bucket
-const destinationFileName = 'latest/text/latest.json'; // The destination file path in your GCS bucket
-
-// Initialize the Google Cloud Storage client
-// const storage = new Storage();
-// const bucket = storage.bucket(bucketName);
-
-// Where to save the merged data
-const filePath = './website/public/data/latest.json';
 
 // Function to fetch data from a given URL
 async function fetchData(url) {
@@ -27,24 +15,6 @@ async function fetchData(url) {
     }
 }
 
-// Function to upload data to Google Cloud Storage
-async function uploadToGCS() {
-    const file = bucket.file(destinationFileName);
-    await file.save(JSON.stringify(data, null, 2), {
-        contentType: 'application/json',
-    });
-    console.log(`"latest.json" has been uploaded successfully to ${destinationFileName}.`);
-}
-
-async function saveData(data) {
-    try {
-        await fs.writeFile(filePath, data, 'utf-8');
-        console.log(`File saved successfully at ${filePath}`);
-    } catch (error) {
-        console.error('Error saving file:', error);
-    }
-}
-
 // Main function to fetch files from all regions and merge them
 async function fetchAndMergeFiles(date) {
     if (!date) {
@@ -65,12 +35,8 @@ async function fetchAndMergeFiles(date) {
         return;
     }
 
-    //@ts-ignore
+    // Output the merged data to the console
     process.stdout.write(JSON.stringify(validResults, null, 2));
-    // await saveData(JSON.stringify(validResults, null, 2));
-
-    // Upload the merged results to Google Cloud Storage
-    // await uploadToGCS(validResults);
 }
 
 // Example usage with the current date

website/src/components/DataGrid.astro

@@ -27,10 +27,11 @@ import 'ag-grid-community/styles/ag-theme-quartz.css';
 <script>
   import { createGrid } from 'ag-grid-community';
   import { BenchmarkRegions, gridOptions, LatestDataUrl } from '@/utils/DataGridDefinitions.ts';
-  import { fetchLatestJsonFile } from '@/utils/FetchData.ts';
+  import { fetchLocalJsonFile } from '@/utils/FetchData.ts';
 
   let gridApi;
-  const gridData = await fetchLatestJsonFile(LatestDataUrl);
+  const localData = '../../data/latest.json';
+  let gridData = [];
 
   function setGridData(region, gridData) {
     gridData.forEach((regionData) => {
@@ -40,7 +41,9 @@ import 'ag-grid-community/styles/ag-theme-quartz.css';
     });
   } 
 
-  function onDOMContentLoaded() {
+  async function onDOMContentLoaded() {
+    gridData = await fetchLocalJsonFile(localData);
+
     // setup the grid after the page has finished loading
     var gridDiv = document.querySelector("#myGrid");
     gridApi = createGrid(gridDiv, gridOptions);

website/src/utils/FetchData.ts

@@ -10,4 +10,18 @@ export async function fetchLatestJsonFile(url: string) {
   } catch (error) {
     console.error('Error fetching JSON file:', error);
   }
+}
+
+export async function fetchLocalJsonFile(url: string) {
+  try {
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const json = await response.json();
+    return json;
+  } catch (error) {
+    console.error('Error fetching JSON file:', error);
+  }
 }