store: add new semi-hidden packaging dashboard (#2315)

* store: add new semi-hidden packaging dashboard

* store/dashboard: be able to filter only favorited apps

store/.gitignore

@@ -1,9 +1,6 @@
 config.toml
 .stars
 .wishlist_ratelimit
-
-assets/fork-awesome.*
-assets/forkawesome-webfont.*
-assets/tailwind.css
-assets/tailwindcss-linux-x64
-assets/ynh_logo_*
+.cache
+.tmp
+assets/*

store/README.md

@@ -42,7 +42,7 @@ It's based on Flask-Babel : <https://python-babel.github.io/flask-babel/>
 source venv/bin/activate
 
 # Extract the english sentences from the code, needed if you modified it
-pybabel extract --ignore-dirs venv -F babel.cfg -o messages.pot .
+pybabel extract -F babel.cfg -o messages.pot *.py templates/*.html
 
 # If working on a new locale: initialize it (in this example: fr)
 pybabel init -i messages.pot -d translations -l fr

store/app.py

@@ -1,15 +1,16 @@
-import time
+import os
+import sys
 import re
+import time
+import json
 import toml
 import tomlkit
 import base64
 import hashlib
 import hmac
-import os
 import string
 import random
 import urllib
-import sys
 from slugify import slugify
 from flask import (
     Flask,
@@ -30,6 +31,7 @@
     get_catalog,
     get_wishlist,
     get_stars,
+    get_dashboard_data,
     get_app_md_and_screenshots,
     save_wishlist_submit_for_ratelimit,
     check_wishlist_submit_ratelimit,
@@ -82,6 +84,11 @@ def localize(d):
             return d["en"]
 
 
+@app.template_filter("days_ago")
+def days_ago(timestamp):
+    return int((time.time() - timestamp) / (60 * 60 * 24))
+
+
 @app.context_processor
 def utils():
     d = {
@@ -446,6 +453,32 @@ def add_to_wishlist():
         )
 
 
+@app.route("/dash")
+def dash():
+    return render_template(
+        "dash.html",
+        data=get_dashboard_data(),
+        stars=get_stars()
+    )
+
+
+@app.route("/charts")
+def charts():
+
+    dashboard_data = get_dashboard_data()
+    level_summary = {}
+    for i in range(0,9):
+        level_summary[i] = len([infos for infos in dashboard_data.values() if infos.get("ci_results", {}).get("main").get("level") == i])
+    level_summary["unknown"] = len([infos for infos in dashboard_data.values() if infos.get("ci_results", {}).get("main").get("level") in [None, "?"]])
+
+    return render_template(
+        "charts.html",
+        level_summary=level_summary,
+        history=json.loads(open(".cache/history.json").read()),
+        news_per_date=json.loads(open(".cache/news.json").read())
+    )
+
+
 ###############################################################################
 #                        Session / SSO using Discourse                        #
 ###############################################################################
@@ -519,6 +552,22 @@ def sso_login_callback():
         return redirect("/")
 
 
+@app.route("/toggle_packaging")
+def toggle_packaging():
+    if session and "user" in session:
+        user = session["user"]
+        if not session["user"].get("packaging_enabled"):
+            # Use this trick to force the change to be registered
+            # because this session["user"]["foobar"] = value doesn't actually change the state ? idk
+            user["packaging_enabled"] = True
+            session["user"] = user
+            return redirect("/dash")
+        else:
+            user["packaging_enabled"] = False
+            session["user"] = user
+    return redirect("/")
+
+
 @app.route("/logout")
 def logout():
     session.clear()

store/assets/fetch_assets

@@ -7,7 +7,10 @@ chmod +x tailwindcss-linux-x64
 # Development -> we use the JS magic thingy
 curl -L https://cdn.tailwindcss.com?plugins=forms > tailwind-css.js
 
-# Forkawesome
+# Canvasjs (for the chart page only)
+curl -L https://cdn.canvasjs.com/ga/canvasjs.min.js > canvasjs.min.js
+
+# Icons / Forkawesome
 curl https://cdn.jsdelivr.net/npm/[email protected]/css/fork-awesome.min.css > fork-awesome.min.css
 sed -i 's@../fonts/@@g' ./fork-awesome.min.css
 curl https://cdn.jsdelivr.net/npm/[email protected]/fonts/forkawesome-webfont.woff2?v=1.2.0 > forkawesome-webfont.woff2

store/config.toml.example

@@ -1,5 +1,7 @@
 COOKIE_SECRET = "abcdefghijklmnopqrstuvwxyz1234567890"
-# This secret is configured in Discourse
+# This secret is configured in Discourse, "discourse connect provider secret"
+# For development, a secret for "localhost" exists.
+# But then be sure to access your dev server using localhost:5000, not 127.0.0.1:5000
 DISCOURSE_SSO_SECRET = "abcdefghijklmnopqrstuvwxyz1234567890"
 DISCOURSE_SSO_ENDPOINT = "https://forum.yunohost.org/session/sso_provider"
 CALLBACK_URL_AFTER_LOGIN_ON_DISCOURSE = "http://localhost:5000/sso_login_callback"

store/fetch_level_history.py

@@ -0,0 +1,200 @@
+import toml
+import json
+import os
+from datetime import datetime
+
+
+def _time_points_until_today():
+
+    year = 2017
+    month = 1
+    day = 1
+    today = datetime.today()
+    date = datetime(year, month, day)
+
+    while date < today:
+        yield date
+
+        day += 14
+        if day > 15:
+            day = 1
+            month += 1
+
+        if month > 12:
+            month = 1
+            year += 1
+
+        date = datetime(year, month, day)
+
+
+time_points_until_today = list(_time_points_until_today())
+
+
+def get_lists_history():
+
+    os.system("rm -rf ./.tmp")
+    os.system("git clone https://github.com/YunoHost/apps ./.tmp/apps")
+
+    for t in time_points_until_today:
+        print(t.strftime("%b %d %Y"))
+
+        # Fetch repo at this date
+        cmd = 'cd ./.tmp/apps; git checkout `git rev-list -1 --before="%s" master`'
+        os.system(cmd % t.strftime("%b %d %Y"))
+
+        if t < datetime(2019, 4, 4):
+            # Merge community and official
+            community = json.loads(open("./.tmp/apps/community.json").read())
+            official = json.loads(open("./.tmp/apps/official.json").read())
+            for key in official:
+                official[key]["state"] = "official"
+            merged = {}
+            merged.update(community)
+            merged.update(official)
+        else:
+            try:
+                merged = toml.loads(open("./.tmp/apps/apps.toml").read())
+            except Exception:
+                try:
+                    merged = json.loads(open("./.tmp/apps/apps.json").read())
+                except Exception:
+                    pass
+
+        # Save it
+        json.dump(
+            merged, open("./.tmp/merged_lists.json.%s" % t.strftime("%y-%m-%d"), "w")
+        )
+
+
+def make_count_summary():
+
+    history = []
+
+    last_time_point = time_points_until_today[-1]
+    json_at_last_time_point = json.loads(
+        open(
+            "./.tmp/merged_lists.json.%s" % last_time_point.strftime("%y-%m-%d")
+        ).read()
+    )
+    relevant_apps_to_track = [
+        app
+        for app, infos in json_at_last_time_point.items()
+        if infos.get("state") in ["working", "official"]
+    ]
+
+    for d in time_points_until_today:
+
+        print("Analyzing %s ..." % d.strftime("%y-%m-%d"))
+
+        # Load corresponding json
+        j = json.loads(
+            open("./.tmp/merged_lists.json.%s" % d.strftime("%y-%m-%d")).read()
+        )
+        d_label = d.strftime("%b %d %Y")
+
+        summary = {}
+        summary["date"] = d_label
+        for level in range(0, 10):
+            summary["level-%s" % level] = len(
+                [
+                    k
+                    for k, infos in j.items()
+                    if infos.get("state") in ["working", "official"]
+                    and infos.get("level", None) == level
+                ]
+            )
+
+        history.append(summary)
+
+        for app in relevant_apps_to_track:
+
+            infos = j.get(app, {})
+
+            if not infos or infos.get("state") not in ["working", "official"]:
+                level = -1
+            else:
+                level = infos.get("level", -1)
+                try:
+                    level = int(level)
+                except Exception:
+                    level = -1
+
+    json.dump(history, open(".cache/history.json", "w"))
+
+
+def make_news():
+
+    news_per_date = {
+        d.strftime("%b %d %Y"): {
+            "broke": [],
+            "repaired": [],
+            "removed": [],
+            "added": [],
+        }
+        for d in time_points_until_today
+    }
+    previous_j = {}
+
+    def level(infos):
+        lev = infos.get("level")
+        if lev is None or (isinstance(lev, str) and not lev.isdigit()):
+            return -1
+        else:
+            return int(lev)
+
+    for d in time_points_until_today:
+        d_label = d.strftime("%b %d %Y")
+
+        print("Analyzing %s ..." % d.strftime("%y-%m-%d"))
+
+        # Load corresponding json
+        j = json.loads(
+            open("./.tmp/merged_lists.json.%s" % d.strftime("%y-%m-%d")).read()
+        )
+
+        apps_current = set(
+            k
+            for k, infos in j.items()
+            if infos.get("state") in ["working", "official"] and level(infos) != -1
+        )
+        apps_current_good = set(
+            k for k, infos in j.items() if k in apps_current and level(infos) > 4
+        )
+        apps_current_broken = set(
+            k for k, infos in j.items() if k in apps_current and level(infos) <= 4
+        )
+
+        apps_previous = set(
+            k
+            for k, infos in previous_j.items()
+            if infos.get("state") in ["working", "official"] and level(infos) != -1
+        )
+        apps_previous_good = set(
+            k
+            for k, infos in previous_j.items()
+            if k in apps_previous and level(infos) > 4
+        )
+        apps_previous_broken = set(
+            k
+            for k, infos in previous_j.items()
+            if k in apps_previous and level(infos) <= 4
+        )
+
+        news = news_per_date[d_label]
+        for app in set(apps_previous_good & apps_current_broken):
+            news["broke"].append((app, j[app]["url"]))
+        for app in set(apps_previous_broken & apps_current_good):
+            news["repaired"].append((app, j[app]["url"]))
+        for app in set(apps_current - apps_previous):
+            news["added"].append((app, j[app]["url"]))
+        for app in set(apps_previous - apps_current):
+            news["removed"].append((app, previous_j[app]["url"]))
+
+        previous_j = j
+
+    json.dump(news_per_date, open(".cache/news.json", "w"))
+
+
+get_lists_history()
+make_count_summary()
+make_news()